{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook will create a desciptive statistics dataset for all players 4 targets.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom numpy import mean,std\nfrom scipy.stats import norm\nimport statistics as st\n\nimport os\nimport gc\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-18T16:40:31.186024Z","iopub.execute_input":"2021-06-18T16:40:31.186628Z","iopub.status.idle":"2021-06-18T16:40:31.967553Z","shell.execute_reply.started":"2021-06-18T16:40:31.186525Z","shell.execute_reply":"2021-06-18T16:40:31.966691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:40:31.968804Z","iopub.execute_input":"2021-06-18T16:40:31.969088Z","iopub.status.idle":"2021-06-18T16:41:48.300119Z","shell.execute_reply.started":"2021-06-18T16:40:31.969062Z","shell.execute_reply":"2021-06-18T16:41:48.299044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#thanks to Alok Pattani https://www.kaggle.com/alokpattani\n\n# Get names of all \"nested\" data frames in daily training set\n#### get all column names\n#daily_data_nested_df_names = train.drop('date', axis = 1).columns.values.tolist()\n\ndaily_data_nested_df_names = ['nextDayPlayerEngagement']\n\nfor df_name in daily_data_nested_df_names:\n    date_nested_table = train[['date', df_name]]\n\n    date_nested_table = (date_nested_table[\n      ~pd.isna(date_nested_table[df_name])\n      ].\n      reset_index(drop = True)\n      )\n    \n    daily_dfs_collection = []\n    \n    for date_index, date_row in date_nested_table.iterrows():\n        daily_df = pd.read_json(date_row[df_name])\n        \n        daily_df['dailyDataDate'] = date_row['date']\n        \n        daily_dfs_collection = daily_dfs_collection + [daily_df]\n\n    # Concatenate all daily dfs into single df for each row\n    unnested_table = (pd.concat(daily_dfs_collection,\n      ignore_index = True).\n      # Set and reset index to move 'dailyDataDate' to front of df\n      set_index('dailyDataDate').\n      reset_index()\n      )\n    \n    # Creates 1 pandas df per unnested df from daily data read in, with same name\n    globals()[df_name] = unnested_table    \n    \n    # Clean up tables and collection of daily data frames for this df\n    del(date_nested_table, daily_dfs_collection, unnested_table)\n\nprint (daily_data_nested_df_names)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:41:48.301738Z","iopub.execute_input":"2021-06-18T16:41:48.302033Z","iopub.status.idle":"2021-06-18T16:42:09.785456Z","shell.execute_reply.started":"2021-06-18T16:41:48.302004Z","shell.execute_reply":"2021-06-18T16:42:09.784492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del(train)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:42:09.786836Z","iopub.execute_input":"2021-06-18T16:42:09.787135Z","iopub.status.idle":"2021-06-18T16:42:09.980171Z","shell.execute_reply.started":"2021-06-18T16:42:09.787105Z","shell.execute_reply":"2021-06-18T16:42:09.979175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nextDayPlayerEngagement","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:42:09.981308Z","iopub.execute_input":"2021-06-18T16:42:09.981575Z","iopub.status.idle":"2021-06-18T16:42:10.006501Z","shell.execute_reply.started":"2021-06-18T16:42:09.981549Z","shell.execute_reply":"2021-06-18T16:42:10.005616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nextDayPlayerEngagement['year'] = pd.DatetimeIndex(nextDayPlayerEngagement['engagementMetricsDate']).year\nnextDayPlayerEngagement['month'] = pd.DatetimeIndex(nextDayPlayerEngagement['engagementMetricsDate']).month","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:42:10.007886Z","iopub.execute_input":"2021-06-18T16:42:10.008307Z","iopub.status.idle":"2021-06-18T16:42:12.21137Z","shell.execute_reply.started":"2021-06-18T16:42:10.008264Z","shell.execute_reply":"2021-06-18T16:42:12.210412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df = nextDayPlayerEngagement[nextDayPlayerEngagement['year'] == 2021]\nnew_df = new_df[new_df['month'] >= 4]\nnew_df","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:42:12.212802Z","iopub.execute_input":"2021-06-18T16:42:12.213185Z","iopub.status.idle":"2021-06-18T16:42:12.302587Z","shell.execute_reply.started":"2021-06-18T16:42:12.213146Z","shell.execute_reply":"2021-06-18T16:42:12.301654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"playerId_list=new_df.playerId.unique().tolist()\n#playerId_list=playerId_list[:10]\n#playerId_list","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:44:19.469788Z","iopub.execute_input":"2021-06-18T16:44:19.470177Z","iopub.status.idle":"2021-06-18T16:44:19.474787Z","shell.execute_reply.started":"2021-06-18T16:44:19.470143Z","shell.execute_reply":"2021-06-18T16:44:19.474044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\ndef calc_probs(pid,df,temp):\n    to_append=[pid,'','','','','','','','','','','','','','','','','','','','','','','','']\n    targets=['target1','target2','target3','target4']\n    z=1\n    for target in targets:\n        target_prob = temp[target].tolist()\n        mean = np.mean(target_prob)\n        std = np.std(target_prob)\n        median = st.median(target_prob)\n        distribution = norm(mean, std)\n        min_weight = min(target_prob)\n        max_weight = max(target_prob)\n        values = list(np.linspace(min_weight, max_weight))\n        probabilities = [distribution.pdf(v) for v in values]\n        max_value = max(probabilities)\n        max_index = probabilities.index(max_value)\n        to_append[z]=mean\n        to_append[z+1]=median\n        to_append[z+2]=std\n        to_append[z+3]=min_weight\n        to_append[z+4]=max_weight\n        to_append[z+5]=target_prob[max_index]\n        z=z+6\n    df_length = len(df)\n    df.loc[df_length] = to_append\n    return df\n    \n\n### CREATE DATAFRAME to store probabilities\ncolumn_names = [\"playerId\", \"target1_mean\",\"target1_median\",\"target1_std\",\"target1_min\",\"target1_max\",\"target1_prob\", \"target2_mean\",\"target2_median\",\"target2_std\",\"target2_min\",\"target2_max\",\"target2_prob\", \"target3_mean\",\"target3_median\",\"target3_std\",\"target3_min\",\"target3_max\",\"target3_prob\", \"target4_mean\",\"target4_median\",\"target4_std\",\"target4_min\",\"target4_max\",\"target4_prob\"]\nplayer_target_probs = pd.DataFrame(columns = column_names)\n    \nfor pid in playerId_list:\n    temp = new_df[new_df['playerId'] == pid]\n    player_target_stats=calc_probs(pid,player_target_probs,temp)\n\nplayer_target_stats","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:44:23.343033Z","iopub.execute_input":"2021-06-18T16:44:23.343527Z","iopub.status.idle":"2021-06-18T16:45:37.051048Z","shell.execute_reply.started":"2021-06-18T16:44:23.343494Z","shell.execute_reply":"2021-06-18T16:45:37.050019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_target_stats.to_csv('player_target_stats.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:45:37.052378Z","iopub.execute_input":"2021-06-18T16:45:37.052659Z","iopub.status.idle":"2021-06-18T16:45:37.158493Z","shell.execute_reply.started":"2021-06-18T16:45:37.052633Z","shell.execute_reply":"2021-06-18T16:45:37.157497Z"},"trusted":true},"execution_count":null,"outputs":[]}]}