{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install pandarallel \n\n# import gc\n\n# import numpy as np\n# import pandas as pd\n# from pathlib import Path\n\n# from pandarallel import pandarallel\n# pandarallel.initialize()\n\n# BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting')\n# train = pd.read_csv(BASE_DIR / 'train.csv')\n\n# null = np.nan\n# true = True\n# false = False\n\n# for col in train.columns:\n\n#     if col == 'date': continue\n\n#     _index = train[col].notnull()\n#     train.loc[_index, col] = train.loc[_index, col].parallel_apply(lambda x: eval(x))\n\n#     outputs = []\n#     for index, date, record in train.loc[_index, ['date', col]].itertuples():\n#         _df = pd.DataFrame(record)\n#         _df['index'] = index\n#         _df['date'] = date\n#         outputs.append(_df)\n\n#     outputs = pd.concat(outputs).reset_index(drop=True)\n\n#     outputs.to_csv(f'{col}_train.csv', index=False)\n#     outputs.to_pickle(f'{col}_train.pkl')\n\n#     del outputs\n#     del train[col]\n#     gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:23.849294Z","iopub.execute_input":"2021-07-07T17:51:23.849979Z","iopub.status.idle":"2021-07-07T17:51:23.855274Z","shell.execute_reply.started":"2021-07-07T17:51:23.849933Z","shell.execute_reply":"2021-07-07T17:51:23.854074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import timedelta\nfrom tqdm import tqdm\nimport mlb\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-07T17:51:23.857130Z","iopub.execute_input":"2021-07-07T17:51:23.857737Z","iopub.status.idle":"2021-07-07T17:51:24.934659Z","shell.execute_reply.started":"2021-07-07T17:51:23.857702Z","shell.execute_reply":"2021-07-07T17:51:24.933473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/players.csv\")\nplayers_stats = pd.read_csv(\"../input/extradata/players_stats.csv\")\nrosters = pd.read_pickle('../input/mlb-pdef-train-dataset/rosters_train.pkl')","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:24.937085Z","iopub.execute_input":"2021-07-07T17:51:24.937415Z","iopub.status.idle":"2021-07-07T17:51:26.407072Z","shell.execute_reply.started":"2021-07-07T17:51:24.937383Z","shell.execute_reply":"2021-07-07T17:51:26.405983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FECOLS = ['t1_mean', 't2_mean', 't3_mean', 't4_mean',\n       't1_max', 't2_max', 't3_max', 't4_max', 't1_min', 't2_min', 't3_min',\n       't4_min', 't1_std', 't2_std', 't3_std', 't4_std']\nTGTCOLS = ['target1', 'target2', 'target3', 'target4']","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:26.409053Z","iopub.execute_input":"2021-07-07T17:51:26.409441Z","iopub.status.idle":"2021-07-07T17:51:26.417396Z","shell.execute_reply.started":"2021-07-07T17:51:26.409395Z","shell.execute_reply":"2021-07-07T17:51:26.416157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:26.419187Z","iopub.execute_input":"2021-07-07T17:51:26.419597Z","iopub.status.idle":"2021-07-07T17:51:32.913850Z","shell.execute_reply.started":"2021-07-07T17:51:26.419543Z","shell.execute_reply":"2021-07-07T17:51:32.912785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model(n_in):\n    inp = L.Input(name=\"inputs\", shape=(n_in,))\n    nh = 128\n    x = L.Dense(nh, activation=\"relu\", name=\"d1\")(inp)\n    x = L.Dense(nh, activation=\"relu\", name=\"d2\")(x)\n    x = L.Dense(nh, activation=\"relu\", name=\"d3\")(x)\n    preds = L.Dense(4, activation=\"linear\", name=\"preds\")(x)\n    \n    model = M.Model(inp, preds, name=\"ANN_XST\")\n    model.compile(loss=\"mean_absolute_error\", optimizer=\"adam\")\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:32.915406Z","iopub.execute_input":"2021-07-07T17:51:32.916058Z","iopub.status.idle":"2021-07-07T17:51:32.926132Z","shell.execute_reply.started":"2021-07-07T17:51:32.915998Z","shell.execute_reply":"2021-07-07T17:51:32.924554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XST = make_model(16)\nprint(XST.summary())","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:32.927856Z","iopub.execute_input":"2021-07-07T17:51:32.928185Z","iopub.status.idle":"2021-07-07T17:51:33.065910Z","shell.execute_reply.started":"2021-07-07T17:51:32.928153Z","shell.execute_reply":"2021-07-07T17:51:33.064774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XST.load_weights(\"../input/trainingxst/model_XST.cpkt\")","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:33.067273Z","iopub.execute_input":"2021-07-07T17:51:33.067582Z","iopub.status.idle":"2021-07-07T17:51:33.180113Z","shell.execute_reply.started":"2021-07-07T17:51:33.067554Z","shell.execute_reply":"2021-07-07T17:51:33.179344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #to debug submission process\n# sample_pred_temp = pd.DataFrame(columns=['date_playerId', 'target1', 'target2', 'target3', 'target4'])\n# test_temp = pd.DataFrame(columns=['date_playerId', 'playerId', 'gameDate', 'teamId', 'statusCode',\n#        'status', 'bmi', 'DOB', 'age', 'is_played', 'Unnamed: 0', 't1_m',\n#        't2_m', 't3_m', 't4_m'])","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:33.182395Z","iopub.execute_input":"2021-07-07T17:51:33.182941Z","iopub.status.idle":"2021-07-07T17:51:33.186359Z","shell.execute_reply.started":"2021-07-07T17:51:33.182907Z","shell.execute_reply":"2021-07-07T17:51:33.185183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"env = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:33.188501Z","iopub.execute_input":"2021-07-07T17:51:33.188830Z","iopub.status.idle":"2021-07-07T17:51:33.201979Z","shell.execute_reply.started":"2021-07-07T17:51:33.188792Z","shell.execute_reply":"2021-07-07T17:51:33.200845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null = np.nan\ntrue = True\nfalse = False\n\nfor (test_df, sample_prediction_df) in iter_test: # make predictions here\n    \n    sample_prediction_df = sample_prediction_df.reset_index(drop=True)\n    \n    # creat dataset\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\\\n                                        .map(lambda x: int(x.split('_')[1]))\n    # Dealing with missing values\n    if test_df['rosters'].iloc[0] == test_df['rosters'].iloc[0]:\n        test_rosters = pd.DataFrame(eval(test_df['rosters'].iloc[0]))\n    else:\n        test_rosters = pd.DataFrame({'playerId': sample_prediction_df['playerId']})\n        for col in rosters.columns:\n            if col == 'playerId': continue\n            test_rosters[col] = np.nan\n            \n    sub_df = sample_prediction_df[['playerId']].copy()\n    sub_df = sub_df.merge(players_stats,on='playerId',how='left')\n    \n    X_fe = sub_df[FECOLS].fillna(0.).values\n    preds = XST.predict(X_fe)\n    preds = np.clip(preds, 0, 100)\n    \n    # merge submission\n    sample_prediction_df[TGTCOLS] =  preds\n    sample_prediction_df = sample_prediction_df.fillna(0.)\n    \n    final_headers = [\"date_playerId\", \"target1\", \"target2\", \"target3\", \"target4\"]\n    sample_prediction_df = sample_prediction_df[final_headers].copy()\n    #del sample_prediction_df['playerId']\n    \n#      #to debug\n#     sample_pred_temp = sample_pred_temp.append(sample_prediction_df)#,ignore_index=True)\n#     #test_temp = test_temp.append(test) #ignore_index=True)\n#     #\n    \n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:33.203304Z","iopub.execute_input":"2021-07-07T17:51:33.203809Z","iopub.status.idle":"2021-07-07T17:51:36.015934Z","shell.execute_reply.started":"2021-07-07T17:51:33.203766Z","shell.execute_reply":"2021-07-07T17:51:36.015056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.017150Z","iopub.execute_input":"2021-07-07T17:51:36.017573Z","iopub.status.idle":"2021-07-07T17:51:36.041366Z","shell.execute_reply.started":"2021-07-07T17:51:36.017541Z","shell.execute_reply":"2021-07-07T17:51:36.040337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # to reset env.predict() to work\n# example_sample_submission = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv\")\n# example_sample_submission\n# #env.predict(example_sample_submission) ","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.042777Z","iopub.execute_input":"2021-07-07T17:51:36.043098Z","iopub.status.idle":"2021-07-07T17:51:36.047360Z","shell.execute_reply.started":"2021-07-07T17:51:36.043060Z","shell.execute_reply":"2021-07-07T17:51:36.046202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # to check if any date_playerId is missing in submission\n# list(example_sample_submission.date_playerId)==list(sample_pred_temp.date_playerId)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.048796Z","iopub.execute_input":"2021-07-07T17:51:36.049178Z","iopub.status.idle":"2021-07-07T17:51:36.060926Z","shell.execute_reply.started":"2021-07-07T17:51:36.049141Z","shell.execute_reply":"2021-07-07T17:51:36.059850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # to trace mismatch\n# for id in zip(sample_pred_temp.date_playerId,example_sample_submission.date_playerId):\n#     if id[0]==id[1]:\n#         pass\n#     else:\n#         print(\"EXCEPTION\")","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.062237Z","iopub.execute_input":"2021-07-07T17:51:36.062619Z","iopub.status.idle":"2021-07-07T17:51:36.072643Z","shell.execute_reply.started":"2021-07-07T17:51:36.062573Z","shell.execute_reply":"2021-07-07T17:51:36.071584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_pred_temp.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.074416Z","iopub.execute_input":"2021-07-07T17:51:36.075114Z","iopub.status.idle":"2021-07-07T17:51:36.090891Z","shell.execute_reply.started":"2021-07-07T17:51:36.075075Z","shell.execute_reply":"2021-07-07T17:51:36.089646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example_sample_submission.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.092443Z","iopub.execute_input":"2021-07-07T17:51:36.092796Z","iopub.status.idle":"2021-07-07T17:51:36.103244Z","shell.execute_reply.started":"2021-07-07T17:51:36.092762Z","shell.execute_reply":"2021-07-07T17:51:36.102356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_pred_temp","metadata":{"execution":{"iopub.status.busy":"2021-07-07T17:51:36.104912Z","iopub.execute_input":"2021-07-07T17:51:36.105237Z","iopub.status.idle":"2021-07-07T17:51:36.115671Z","shell.execute_reply.started":"2021-07-07T17:51:36.105207Z","shell.execute_reply":"2021-07-07T17:51:36.114381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}