{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-16T04:16:17.921233Z","iopub.execute_input":"2021-07-16T04:16:17.921789Z","iopub.status.idle":"2021-07-16T04:16:17.946466Z","shell.execute_reply.started":"2021-07-16T04:16:17.921729Z","shell.execute_reply":"2021-07-16T04:16:17.945687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom datetime import timedelta\nimport warnings\nimport gc\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:17.947642Z","iopub.execute_input":"2021-07-16T04:16:17.948023Z","iopub.status.idle":"2021-07-16T04:16:17.953048Z","shell.execute_reply.started":"2021-07-16T04:16:17.947996Z","shell.execute_reply":"2021-07-16T04:16:17.952140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def json_to_df(df, column):\n    num_rows = len(df)\n    data_list = []\n    for row in tqdm(range(num_rows)):\n\n        json_data = df.iloc[row][column]\n        if str(json_data) != \"nan\":\n            data = pd.read_json(json_data)\n            data_list.append(data)\n    all_data = pd.concat(data_list, axis = 0)\n    \n    return all_data","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:17.954987Z","iopub.execute_input":"2021-07-16T04:16:17.955319Z","iopub.status.idle":"2021-07-16T04:16:17.964660Z","shell.execute_reply.started":"2021-07-16T04:16:17.955291Z","shell.execute_reply":"2021-07-16T04:16:17.963696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfile_path = '/kaggle/input/mlb-player-digital-engagement-forecasting/'\n# train = pd.read_csv(file_path + 'train.csv')\ntest = pd.read_csv(file_path + 'example_test.csv')\nsubmit = pd.read_csv(file_path + 'example_sample_submission.csv')\nplayer_engagement = pd.read_pickle('/kaggle/input/mlb-player-engagement/train_player_engagement.pkl')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:17.966114Z","iopub.execute_input":"2021-07-16T04:16:17.966567Z","iopub.status.idle":"2021-07-16T04:16:20.124894Z","shell.execute_reply.started":"2021-07-16T04:16:17.966532Z","shell.execute_reply":"2021-07-16T04:16:20.123992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# player_engagement = json_to_df(train, 'nextDayPlayerEngagement')\n# player_engagement.insert(0, 'date', pd.to_datetime(player_engagement['engagementMetricsDate'])-\\\n#                                                    timedelta(days=1))\n# player_engagement['engagementMetricsDate'] = pd.to_datetime(player_engagement['engagementMetricsDate'])\n# player_engagement.reset_index(drop=True, inplace=True)\n# player_engagement.engagementMetricsDate = player_engagement.engagementMetricsDate.astype(str)\n# year = list(map(lambda x: x.split('-')[0] ,player_engagement.engagementMetricsDate))\n# month = list(map(lambda x: x.split('-')[1] ,player_engagement.engagementMetricsDate))\n# day = list(map(lambda x: x.split('-')[2] ,player_engagement.engagementMetricsDate))\n# player_engagement['date_'] = list(map(lambda x,y,z: x+y+z,year,month,day ))\n# player_engagement['date_playerId'] = list(map(lambda x,y: x+'_'+y,player_engagement.date_,player_engagement.playerId.astype(str)))\n# player_engagement.drop('date_',axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:20.126614Z","iopub.execute_input":"2021-07-16T04:16:20.127026Z","iopub.status.idle":"2021-07-16T04:16:20.131596Z","shell.execute_reply.started":"2021-07-16T04:16:20.126984Z","shell.execute_reply":"2021-07-16T04:16:20.130585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_engagement['year'] = player_engagement.engagementMetricsDate.astype('datetime64').dt.year\nplayer_engagement['month'] = player_engagement.engagementMetricsDate.astype('datetime64').dt.month\nplayer_engagement['day'] = player_engagement.engagementMetricsDate.astype('datetime64').dt.day","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:20.133576Z","iopub.execute_input":"2021-07-16T04:16:20.134065Z","iopub.status.idle":"2021-07-16T04:16:22.598092Z","shell.execute_reply.started":"2021-07-16T04:16:20.134023Z","shell.execute_reply":"2021-07-16T04:16:22.597287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid = player_engagement[(player_engagement.year==2021)&(player_engagement.month>=4)]\ntrain = player_engagement.drop(valid.index)\nvalid.reset_index(drop=True,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:22.601318Z","iopub.execute_input":"2021-07-16T04:16:22.601675Z","iopub.status.idle":"2021-07-16T04:16:22.893784Z","shell.execute_reply.started":"2021-07-16T04:16:22.601647Z","shell.execute_reply":"2021-07-16T04:16:22.892885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = train.groupby(['playerId','year'])['target1'].agg(target1_mean='mean',\n                                                                                        target1_std='std',\n                                                                                                 target1_median='median',\n                                                                                                 target1_max='max',\n                                                                                                 target1_sum='sum',\n                                                                                                 target1_min='min').reset_index()\ntarget2 = train.groupby(['playerId','year'])['target2'].agg(target2_mean='mean',\n                                                                                        target2_std='std',\n                                                                                                 target2_median='median',\n                                                                                                 target2_max='max',\n                                                                                                 target2_sum='sum',\n                                                                                                 target2_min='min').reset_index()\ntarget3 = train.groupby(['playerId','year'])['target3'].agg(target3_mean='mean',\n                                                                                        target3_std='std',\n                                                                                                 target3_median='median',\n                                                                                                 target3_max='max',\n                                                                                                 target3_sum='sum',\n                                                                                                 target3_min='min').reset_index()\ntarget4 = train.groupby(['playerId','year'])['target4'].agg(target4_mean='mean',\n                                                                                        target4_std='std',\n                                                                                                 target4_median='median',\n                                                                                                 target4_sum='sum',\n                                                                                                 target4_max='max',\n                                                                                                 target4_min='min').reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:22.896906Z","iopub.execute_input":"2021-07-16T04:16:22.897350Z","iopub.status.idle":"2021-07-16T04:16:23.958953Z","shell.execute_reply.started":"2021-07-16T04:16:22.897307Z","shell.execute_reply":"2021-07-16T04:16:23.958153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train[['playerId','year','month','day']]\ntrain_label = train[['target1','target2','target3','target4']]\nfor target in [target1,target2,target3,target4]:\n        train_data = pd.merge(train_data,target,on=['year','playerId'],how='left')\n        \ntest_data = valid[['playerId','year','month','day']]\ntest_label = valid[['target1','target2','target3','target4']]\nfor target in [target1,target2,target3,target4]:\n        test_data = pd.merge(test_data,target,on=['year','playerId'],how='left') ","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:23.960306Z","iopub.execute_input":"2021-07-16T04:16:23.960600Z","iopub.status.idle":"2021-07-16T04:16:25.777268Z","shell.execute_reply.started":"2021-07-16T04:16:23.960575Z","shell.execute_reply":"2021-07-16T04:16:25.776367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nfrom lightgbm import LGBMRegressor\n\nALL_LOSS = 0\nNUM_TREE = []\nmodel = LGBMRegressor(n_estimators=1000,random_state=2020)\nfor target in ['target1','target2','target3','target4']:\n    mae_loss = 0\n    model.fit(train_data,train_label[target],eval_set=(test_data,test_label[target]),early_stopping_rounds=50,eval_metric='mae',verbose=False,\n             categorical_feature=['playerId'])\n    mae_loss = mean_absolute_error(model.predict(test_data),test_label[target])\n        \n    ALL_LOSS += mae_loss\n    NUM_TREE.append(model.best_iteration_)\n    print('mae_loss:',mae_loss)\nprint('finally loss:',ALL_LOSS/4)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:16:25.778461Z","iopub.execute_input":"2021-07-16T04:16:25.778729Z","iopub.status.idle":"2021-07-16T04:17:17.283499Z","shell.execute_reply.started":"2021-07-16T04:16:25.778704Z","shell.execute_reply":"2021-07-16T04:17:17.282527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = player_engagement.groupby(['playerId','year'])['target1'].agg(target1_mean='mean',target1_std='std',\n                                                            target1_median='median',target1_max='max',\n                                                            target1_sum='sum',target1_min='min').reset_index()\ntarget2 = player_engagement.groupby(['playerId','year'])['target2'].agg(target2_mean='mean',target2_std='std',\n                                                            target2_median='median',target2_max='max',\n                                                            target2_sum='sum',target2_min='min').reset_index()\ntarget3 = player_engagement.groupby(['playerId','year'])['target3'].agg(target3_mean='mean',target3_std='std',\n                                                            target3_median='median',target3_max='max',\n                                                            target3_sum='sum',target3_min='min').reset_index()\ntarget4 = player_engagement.groupby(['playerId','year'])['target4'].agg(target4_mean='mean',target4_std='std',\n                                                            target4_median='median',target4_max='max',\n                                                            target4_sum='sum',target4_min='min').reset_index()\ndata = player_engagement[['playerId','year','month','day']]\nlabel = player_engagement[['target1','target2','target3','target4']]\nfor target in [target1,target2,target3,target4]:\n        data = pd.merge(data,target,on=['year','playerId'],how='left')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:17:17.287067Z","iopub.execute_input":"2021-07-16T04:17:17.287573Z","iopub.status.idle":"2021-07-16T04:17:20.068331Z","shell.execute_reply.started":"2021-07-16T04:17:17.287533Z","shell.execute_reply":"2021-07-16T04:17:20.067531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = LGBMRegressor(n_estimators=NUM_TREE[0],random_state=2020)\nmodel2 = LGBMRegressor(n_estimators=NUM_TREE[1],random_state=2020)\nmodel3 = LGBMRegressor(n_estimators=NUM_TREE[2],random_state=2020)\nmodel4 = LGBMRegressor(n_estimators=NUM_TREE[3],random_state=2020)\nmodel1.fit(data,label['target1'],eval_set=(data,label['target1']),eval_metric='mae',verbose=False,\n             categorical_feature=['playerId'])\nmodel2.fit(data,label['target2'],eval_set=(data,label['target2']),eval_metric='mae',verbose=False,\n             categorical_feature=['playerId'])\nmodel3.fit(data,label['target3'],eval_set=(data,label['target3']),eval_metric='mae',verbose=False,\n             categorical_feature=['playerId'])\nmodel4.fit(data,label['target4'],eval_set=(data,label['target4']),eval_metric='mae',verbose=False,\n             categorical_feature=['playerId'])","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:17:20.069606Z","iopub.execute_input":"2021-07-16T04:17:20.070144Z","iopub.status.idle":"2021-07-16T04:17:46.591845Z","shell.execute_reply.started":"2021-07-16T04:17:20.070101Z","shell.execute_reply":"2021-07-16T04:17:46.591041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data,label,train_data,train_label,test_data,test_label,train,player_engagement\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:17:46.595361Z","iopub.execute_input":"2021-07-16T04:17:46.595642Z","iopub.status.idle":"2021-07-16T04:17:47.155168Z","shell.execute_reply.started":"2021-07-16T04:17:46.595616Z","shell.execute_reply":"2021-07-16T04:17:47.154320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:17:47.156251Z","iopub.execute_input":"2021-07-16T04:17:47.156521Z","iopub.status.idle":"2021-07-16T04:17:47.296940Z","shell.execute_reply.started":"2021-07-16T04:17:47.156495Z","shell.execute_reply":"2021-07-16T04:17:47.295671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    submit = pd.DataFrame()\n    submit['playerId'] = sample_prediction_df['date_playerId'].str.split('_',expand=True)[1].astype('int64')\n    submit['date'] = sample_prediction_df['date_playerId'].str.split('_',expand=True)[0].astype('int64')    \n    submit['year'] = submit.date//10000\n    submit['month'] = submit.date//100%100\n    submit['day'] = submit.date%100\n    for target in [target1,target2,target3,target4]:\n        submit = pd.merge(submit,target,on=['year','playerId'],how='left')\n    submit = submit.drop('date',axis=1)\n    sample_prediction_df['target1'] = np.clip(model1.predict(submit),0,100) #make predictions here\n    sample_prediction_df['target2'] = np.clip(model2.predict(submit),0,100)\n    sample_prediction_df['target3'] = np.clip(model3.predict(submit),0,100)\n    sample_prediction_df['target4'] = np.clip(model4.predict(submit),0,100)\n    sample_prediction_df = sample_prediction_df.fillna(0)\n    sample_prediction_df = sample_prediction_df.reset_index(drop=True)\n    print(sample_prediction_df)\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:17:47.297955Z","iopub.status.idle":"2021-07-16T04:17:47.298394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}