{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-16T14:22:45.964971Z","iopub.execute_input":"2021-07-16T14:22:45.965520Z","iopub.status.idle":"2021-07-16T14:22:45.984167Z","shell.execute_reply.started":"2021-07-16T14:22:45.965437Z","shell.execute_reply":"2021-07-16T14:22:45.982997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Function To reduce the size of the data***","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float64)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:23:53.636264Z","iopub.execute_input":"2021-07-16T14:23:53.636729Z","iopub.status.idle":"2021-07-16T14:23:53.741354Z","shell.execute_reply.started":"2021-07-16T14:23:53.636670Z","shell.execute_reply":"2021-07-16T14:23:53.740285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Reading Data**","metadata":{}},{"cell_type":"code","source":"%%time\nplayers = pd.read_csv('/kaggle/input/mlb-player-digital-engagement-forecasting/players.csv')\nplayers = reduce_mem_usage(players,verbose = False)\n\nteams = pd.read_csv('/kaggle/input/mlb-player-digital-engagement-forecasting/teams.csv')\nteams = reduce_mem_usage(teams,verbose = False)\n\nseasons = pd.read_csv('/kaggle/input/mlb-player-digital-engagement-forecasting/seasons.csv')\nseasons = reduce_mem_usage(seasons,verbose = False)\n\ntrain = pd.read_csv('/kaggle/input/mlb-player-digital-engagement-forecasting/train.csv')\ntrain = reduce_mem_usage(train,verbose = False)\n\nawards = pd.read_csv('/kaggle/input/mlb-player-digital-engagement-forecasting/awards.csv')\nawards = reduce_mem_usage(awards,verbose = False)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:29:24.998975Z","iopub.execute_input":"2021-07-16T14:29:24.999377Z","iopub.status.idle":"2021-07-16T14:30:49.595763Z","shell.execute_reply.started":"2021-07-16T14:29:24.999345Z","shell.execute_reply":"2021-07-16T14:30:49.594508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pids_test = players.playerId[players.playerForTestSetAndFuturePreds == True]","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:30:49.597958Z","iopub.execute_input":"2021-07-16T14:30:49.598280Z","iopub.status.idle":"2021-07-16T14:30:49.606420Z","shell.execute_reply.started":"2021-07-16T14:30:49.598246Z","shell.execute_reply":"2021-07-16T14:30:49.605450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_raw_data(raw_data,dfs_name):\n    \n    unnested_data_dict = dict()\n# columns = train.drop('date', axis = 1).columns.values.tolist()\n# columns\n\n    for col in dfs_name:\n\n        data_nested_info = raw_data[['date',col]]\n\n        data_nested_info = (data_nested_info[\n              ~pd.isna(data_nested_info[col])\n              ].\n              reset_index(drop = True)\n              )\n\n        daily_dfs_collection = []\n        for data_index, data_row in data_nested_info.iterrows():\n            daily_df = pd.read_json(data_row[col])\n\n            daily_df['dailydate'] = data_row['date']\n\n            daily_dfs_collection = daily_dfs_collection + [daily_df]\n\n        unnested_table = (pd.concat(daily_dfs_collection,\n              ignore_index = True).\n                # Set and reset index to move 'dailyDataDate' to front of df\n              set_index('dailydate').\n              reset_index()\n              )\n\n    #     display(col)\n        unnested_table = reduce_mem_usage(unnested_table,verbose = False)\n\n        unnested_data_dict[col] = unnested_table\n        del daily_dfs_collection,unnested_table\n    return unnested_data_dict \n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:33:51.970791Z","iopub.execute_input":"2021-07-16T14:33:51.971201Z","iopub.status.idle":"2021-07-16T14:33:51.980287Z","shell.execute_reply.started":"2021-07-16T14:33:51.971169Z","shell.execute_reply":"2021-07-16T14:33:51.979355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check = unpack_raw_data(train,['playerBoxScores','games'])\nfeatures = ['dailydate','engagementMetricsDate','target1','target2','target3','target4','flyOuts','strikeOuts','stolenBases','homeRunsPitching']","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:33:55.645403Z","iopub.execute_input":"2021-07-16T14:33:55.646444Z","iopub.status.idle":"2021-07-16T14:33:55.652832Z","shell.execute_reply.started":"2021-07-16T14:33:55.646379Z","shell.execute_reply":"2021-07-16T14:33:55.651571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_train_data(raw_data,features):\n    \n    nday_pl_eng = raw_data['nextDayPlayerEngagement']\n    pl_box_scores = raw_data['playerBoxScores']\n  \n    pl_eng_w_scores = pd.merge(nday_pl_eng,pl_box_scores,on=['dailydate','playerId'],how = 'inner')\n    train_data = pl_eng_w_scores[features]\n    del nday_pl_eng,pl_box_scores\n    return train_data,features","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:33:57.679049Z","iopub.execute_input":"2021-07-16T14:33:57.679540Z","iopub.status.idle":"2021-07-16T14:33:57.686398Z","shell.execute_reply.started":"2021-07-16T14:33:57.679488Z","shell.execute_reply":"2021-07-16T14:33:57.684927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_data = unpack_raw_data(train,['nextDayPlayerEngagement','playerBoxScores'])\ntrain_data,features = make_train_data(raw_data,features)\ndel(raw_data)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:34:00.058853Z","iopub.execute_input":"2021-07-16T14:34:00.059223Z","iopub.status.idle":"2021-07-16T14:34:56.752520Z","shell.execute_reply.started":"2021-07-16T14:34:00.059192Z","shell.execute_reply":"2021-07-16T14:34:56.751382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.fillna(-1,inplace = True)\nsample_y = train_data[['target1','target2','target3','target4']]\nsample_X = train_data[['flyOuts','strikeOuts','stolenBases','homeRunsPitching']]\n\n# display(sample_X.head())\n# display(sample_y.head())\nfrom sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(sample_X,sample_y)\ndel train_data,sample_X,sample_y","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:37:03.072393Z","iopub.execute_input":"2021-07-16T14:37:03.072951Z","iopub.status.idle":"2021-07-16T14:37:04.766078Z","shell.execute_reply.started":"2021-07-16T14:37:03.072903Z","shell.execute_reply":"2021-07-16T14:37:04.764924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Building ANN and training the model***","metadata":{}},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nmodel = Sequential()\nmodel.add(Dense(6,input_dim = 4,activation = 'relu'))\nmodel.add(Dense(12,activation = 'relu'))\nmodel.add(Dense(12,activation = 'relu'))\nmodel.add(Dense(4))\nmodel.compile(optimizer = 'adam',loss = 'mae', metrics = ['mae'])\n\nfit_model = model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=100,batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:38:31.499126Z","iopub.execute_input":"2021-07-16T14:38:31.499582Z","iopub.status.idle":"2021-07-16T14:43:52.793841Z","shell.execute_reply.started":"2021-07-16T14:38:31.499543Z","shell.execute_reply":"2021-07-16T14:43:52.792908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['flyOuts','strikeOuts','stolenBases','homeRunsPitching']\nprimary_cols = 'playerBoxScores'","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:43:52.795941Z","iopub.execute_input":"2021-07-16T14:43:52.796575Z","iopub.status.idle":"2021-07-16T14:43:52.802372Z","shell.execute_reply.started":"2021-07-16T14:43:52.796517Z","shell.execute_reply":"2021-07-16T14:43:52.801180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_features(unnested_data_dict,primary_cols,features,sample_prediction_df):\n    \n    test_set = unnested_data_dict[primary_cols]\n    tmp = features.copy()\n    tmp.append('playerId')\n    test_set = test_set[tmp]\n    test_set = test_set.groupby('playerId').sum().reset_index()\n    test_set = test_set.merge(pids_test,on = 'playerId',how = 'right')\n    test_set = test_set.fillna(-1)\n    sub_df = sample_prediction_df.copy()\n    sub_df['playerId'] = sub_df['date_playerId'].map(lambda x: int(x.split('_')[1]))\n    test_set = sub_df.merge(test_set,on = 'playerId', how = 'left')\n    test_set = test_set[features]\n\n    return test_set ","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:43:52.804384Z","iopub.execute_input":"2021-07-16T14:43:52.804843Z","iopub.status.idle":"2021-07-16T14:43:52.823716Z","shell.execute_reply.started":"2021-07-16T14:43:52.804798Z","shell.execute_reply":"2021-07-16T14:43:52.822779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Submission***","metadata":{}},{"cell_type":"code","source":"import mlb\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    test_df = test_df.reset_index().rename(columns = {'index':'date'})\n    test_data = unpack_raw_data(test_df,['playerBoxScores'])\n    \n    fit_data = make_features(test_data,primary_cols,features,sample_prediction_df)\n    \n    pred = model.predict(fit_data)\n    pred = pred.clip(0,100)\n    pred = pred.round(2)\n    \n    sample_prediction_df[['target1','target2','target3','target4']] = pred\n                        \n    \n    \n#     sample_prediction_df['target1'] = 0.4\n#     sample_prediction_df['target2'] = 2\n#     sample_prediction_df['target3'] = 0.4\n#     sample_prediction_df['target4'] = 0.7\n    \n#     sample_prediction_df = sample_prediction_df[['date_playerId']].reset_index().merge(submission,\n#                                 how='left', on='date_playerId').set_index('date')\n#     del submission\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T14:43:52.825233Z","iopub.execute_input":"2021-07-16T14:43:52.825924Z","iopub.status.idle":"2021-07-16T14:43:55.954053Z","shell.execute_reply.started":"2021-07-16T14:43:52.825877Z","shell.execute_reply":"2021-07-16T14:43:55.952835Z"},"trusted":true},"execution_count":null,"outputs":[]}]}