{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport plotly.express as px\nimport ipywidgets as widgets\nimport gc\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-28T17:43:32.842979Z","iopub.execute_input":"2021-06-28T17:43:32.843627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/players.csv')\nseasons = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/seasons.csv')\nawards = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/awards.csv')\nteams = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/teams.csv')\ntrain = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train.csv')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def The_unpack_json_func(json_str):\n    return np.nan if pd.isna(json_str) else pd.read_json(json_str)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Dataset Shape: \",train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*  date:No null values with incorrect data type it should be datetime. \n*  nextDayPlayerEngagement:nested json format \n*  games:nested json format.\n*  rosters:nested json format. \n*  playerBoxScores:  nested json format.\n*  teamBoxScores: nested json format. \n*  transactions:nested json format.  \n*   standings:nested json format.\n*  awards:nested json format****.\n*  events:nested json format.\n*  playerTwitterFollowers:nested json format.\n* teamTwitterFollowers:nested json format","metadata":{}},{"cell_type":"markdown","source":"here we notice that the datatype for date should be date and not int64.","metadata":{}},{"cell_type":"code","source":"train['date']=pd.to_datetime(train['date'],format=\"%Y%m%d\")\ntrain.head()\ntype(train['date'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before moving I want to know what I want o predict from the data .So for that I will first go through sample submission file.","metadata":{}},{"cell_type":"code","source":"example_sample_submission = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv\")\nexample_sample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"going through this info,I think that in this project we need to predict the future fan engagement(this is also written in the data info available at kaggle) such as \"reaction\" and \"behaviour\" with the digital content of each NLB player ID . We will predict four different measures of engagement (target1-target4), each quantified on a scale of 0-100 with a different index.","metadata":{}},{"cell_type":"markdown","source":" Guess from what information? ( looking  through the test data first)","metadata":{}},{"cell_type":"code","source":"example_test = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/example_test.csv\")\nexample_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_test.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize the example_test[\"games\"].iloc[0]","metadata":{}},{"cell_type":"code","source":"The_unpack_json_func(example_test[\"games\"].iloc[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualise for roasters.iloc[0]","metadata":{}},{"cell_type":"code","source":"The_unpack_json_func(example_test[\"rosters\"].iloc[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the info in this area ,I think its the estimate about the competition regarding the estimation of the expected values of the targets1-4 on the next day for each player.","metadata":{}},{"cell_type":"markdown","source":"Coming back to training data","metadata":{}},{"cell_type":"markdown","source":"the contents of train.csv are also in json file format in one cell, and it has a complicated shape like dataframe is further contained.So lets uncomplicate it.","metadata":{}},{"cell_type":"code","source":"def opencol(col,n):\n    tmp = train[col]\n    tmp = tmp.dropna()\n    tmpdf = The_unpack_json_func(tmp.iloc[n])\n    print(tmpdf.columns)\n    return tmpdf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_next_day = opencol(\"nextDayPlayerEngagement\",0)\ntrain_next_day","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_next_day.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_date = list(train_next_day.engagementMetricsDate.unique())\ntarget1_lis = []\ntarget2_lis = []\ntarget3_lis = []\ntarget4_lis = []\nfor i in unique_date:\n    df = train_next_day[train_next_day['engagementMetricsDate']==i]\n    target1 = (df['target1'].sum())/len(df)\n    target2 = (df['target2'].sum())/len(df)\n    target3 = (df['target3'].sum())/len(df)\n    target4 = (df['target4'].sum())/len(df)\n    target1_lis.append(target1)\n    target2_lis.append(target2)\n    target3_lis.append(target3)\n    target4_lis.append(target4)\ntarget1_lis\ntarget2_lis\n   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npx.line(df,x=unique_date,y=target1_lis,title='target 1 over time')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\npx.line(x=unique_date,y=target2_lis,title='target 2 over time')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nfig = px.line(x=unique_date, y=target2_lis)\niplot(fig)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\npx.line(df,x=unique_date,y=target4_lis,title='target 4 over time')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"team_twitter = opencol(\"teamTwitterFollowers\",0)\nteam_twitter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"standings = opencol(\"standings\",0)\ntransactions = opencol(\"transactions\",0)\nstandings","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = transactions.dropna()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_transaction_team=transactions.groupby('fromTeamName')['toTeamName'].count().reset_index(name='Count').sort_values('Count',ascending=False)\npx.bar(most_transaction_team.head(10),x='fromTeamName',y='Count')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_transaction_player=transactions.groupby('playerName')['toTeamName'].count().reset_index(name='Count').sort_values('Count',ascending=False)\npx.bar(most_transaction_player.head(10),x='playerName',y='Count')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(players,x='birthCountry',color='birthCountry')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = \"../input/mlb-player-digital-engagement-forecasting\"\ndf_names=['seasons','teams','players','awards']\nkaggle_data_tabs = widgets.Tab()\n# Display each DataFrame in widgets as a tab inside children\nkaggle_data_tabs.children = list([widgets.Output() for df_name in df_names])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(df_names)):\n    kaggle_data_tabs.set_title(i, df_names[i])\n    \n    df = pd.read_csv(os.path.join(data_dir,df_names[i]) + \".csv\")\n    \n    # Embed DataFrame in each tab\n    with kaggle_data_tabs.children[i]:\n        display(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_data_tabs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seasons = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/seasons.csv\")\nseasons\nseasons['seasonStartDate']=pd.to_datetime(seasons['seasonStartDate'])\nseasons['seasonEndDate']=pd.to_datetime(seasons['seasonEndDate'])\nseasons['preSeasonStartDate']=pd.to_datetime(seasons['preSeasonStartDate'])\nseasons['preSeasonEndDate']=pd.to_datetime(seasons['preSeasonEndDate'])\nseasons['regularSeasonStartDate']=pd.to_datetime(seasons['regularSeasonStartDate'])\nseasons['regularSeasonEndDate']=pd.to_datetime(seasons['regularSeasonEndDate'])\nseasons['lastDate1stHalf']=pd.to_datetime(seasons['lastDate1stHalf'])\nseasons['allStarDate']=pd.to_datetime(seasons['allStarDate'])\nseasons['firstDate2ndHalf']=pd.to_datetime(seasons['firstDate2ndHalf'])\nseasons['postSeasonStartDate']=pd.to_datetime(seasons['postSeasonStartDate'])\nseasons['postSeasonEndDate']=pd.to_datetime(seasons['seasonStartDate'])\nseasons\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*  seasonId\n*  seasonStartDate\n* seasonEndDate\n* preSeasonStartDate\n* preSeasonEndDate\n* regularSeasonStartDate\n* regularSeasonEndDate\n* lastDate1stHalf\n* allStarDate\n* firstDate2ndHalf\n* postSeasonStartDate\n* postSeasonEndDate\n","metadata":{}},{"cell_type":"code","source":"players = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/players.csv\")\nplayers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"awards = pd.read_csv(\"../input/mlb-player-digital-engagement-forecasting/seasons.csv\")\nawards","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in df_names:\n    globals()[name] = pd.read_csv(os.path.join(data_dir,name)+ \".csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_data_unnested_dfs = pd.DataFrame(data = {\n  'dfName': train.drop('date', axis = 1).columns.values.tolist()\n  })\ndaily_data_unnested_dfs['df'] = [pd.DataFrame() for row in \n  daily_data_unnested_dfs.iterrows()]\nfor df_index,df_row in daily_data_unnested_dfs.iterrows():\n    nesteddatatableName= str(df_row['dfName'])\n    date_nested_table=train[['date',nesteddatatableName]]\n    #Removing the null values\n    date_nested_table=(date_nested_table[~pd.isna(date_nested_table[nesteddatatableName])].reset_index(drop=True))\n    daily_dfs_list=[]\n    for date_index,date_row in date_nested_table.iterrows():\n        daily_df= The_unpack_json_func(date_row[nesteddatatableName])\n        daily_df['dailyDataDate'] = date_row['date']\n        daily_dfs_list=daily_dfs_list+[daily_df]\n    \n    unnested_table = pd.concat(daily_dfs_list,ignore_index = True).set_index('dailyDataDate').reset_index()\n        \n# Creates 1 pandas df per unnested df from daily data read in, with same name\n    globals()[df_row['dfName']] = unnested_table    \n    \n    daily_data_unnested_dfs['df'][df_index] = unnested_table\n\ndel train\ngc.collect\n  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_data_unnested_dfs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### Get some information on each date in daily data (using season dates of interest) ####\ndates = pd.DataFrame(data = \n  {'dailyDataDate': nextDayPlayerEngagement['dailyDataDate'].unique()})\n\ndates['date'] = pd.to_datetime(dates['dailyDataDate'].astype(str))\n\ndates['year'] = dates['date'].dt.year\ndates['month'] = dates['date'].dt.month\n\ndates_with_info = pd.merge(\n  dates,\n  seasons,\n  left_on = 'year',\n  right_on = 'seasonId'\n  )\ndates_with_info['inSeason'] = (\n  dates_with_info['date'].between(\n    dates_with_info['regularSeasonStartDate'],\n    dates_with_info['postSeasonEndDate'],\n    inclusive = True\n    )\n  )\ndates_with_info['seasonPart'] = np.select(\n  [\n    dates_with_info['date'] < dates_with_info['preSeasonStartDate'], \n    dates_with_info['date'] < dates_with_info['regularSeasonStartDate'],\n    dates_with_info['date'] <= dates_with_info['lastDate1stHalf'],\n    dates_with_info['date'] < dates_with_info['firstDate2ndHalf'],\n    dates_with_info['date'] <= dates_with_info['regularSeasonEndDate'],\n    dates_with_info['date'] < dates_with_info['postSeasonStartDate'],\n    dates_with_info['date'] <= dates_with_info['postSeasonEndDate'],\n    dates_with_info['date'] > dates_with_info['postSeasonEndDate']\n  ], \n  [\n    'Offseason',\n    'Preseason',\n    'Reg Season 1st Half',\n    'All-Star Break',\n    'Reg Season 2nd Half',\n    'Between Reg and Postseason',\n    'Postseason',\n    'Offseason'\n  ], \n  default = np.nan\n  )\n#### Add some pitching stats/pieces of info to player game level stats ####\n\nplayer_game_stats = (playerBoxScores.copy().\n  # Change team Id/name to reflect these come from player game, not roster\n  rename(columns = {'teamId': 'gameTeamId', 'teamName': 'gameTeamName'})\n  )\n\n# Adds in field for innings pitched as fraction (better for aggregation)\nplayer_game_stats['inningsPitchedAsFrac'] = np.where(\n  pd.isna(player_game_stats['inningsPitched']),\n  np.nan,\n  np.floor(player_game_stats['inningsPitched']) +\n    (player_game_stats['inningsPitched'] -\n      np.floor(player_game_stats['inningsPitched'])) * 10/3\n  )\n\n# Add in Tom Tango pitching game score (https://www.mlb.com/glossary/advanced-stats/game-score)\nplayer_game_stats['pitchingGameScore'] = (40\n#     + 2 * player_game_stats['outs']\n    + 1 * player_game_stats['strikeOutsPitching']\n    - 2 * player_game_stats['baseOnBallsPitching']\n    - 2 * player_game_stats['hitsPitching']\n    - 3 * player_game_stats['runsPitching']\n    - 6 * player_game_stats['homeRunsPitching']\n    )\n# Add in criteria for no-hitter by pitcher (individual, not multiple pitchers)\nplayer_game_stats['noHitter'] = np.where(\n  (player_game_stats['gamesStartedPitching'] == 1) &\n  (player_game_stats['inningsPitched'] >= 9) &\n  (player_game_stats['hitsPitching'] == 0),\n  1, 0\n  )\n\nplayer_date_stats_agg = pd.merge(\n  (player_game_stats.\n    groupby(['dailyDataDate', 'playerId'], as_index = False).\n    # Some aggregations that are not simple sums\n    agg(\n      numGames = ('gamePk', 'nunique'),\n      # Should be 1 team per player per day, but adding here for 1 exception:\n      # playerId 518617 (Jake Diekman) had 2 games for different teams marked\n      # as played on 5/19/19, due to resumption of game after he was traded\n      numTeams = ('gameTeamId', 'nunique'),\n      # Should be only 1 team for almost all player-dates, taking min to simplify\n      gameTeamId = ('gameTeamId', 'min')\n      )\n    ),\n  # Merge with a bunch of player stats that can be summed at date/player level\n  (player_game_stats.\n    groupby(['dailyDataDate', 'playerId'], as_index = False)\n    [['runsScored', 'homeRuns', 'strikeOuts', 'baseOnBalls', 'hits',\n      'hitByPitch', 'atBats', 'caughtStealing', 'stolenBases',\n      'groundIntoDoublePlay', 'groundIntoTriplePlay', 'plateAppearances',\n      'totalBases', 'rbi', 'leftOnBase', 'sacBunts', 'sacFlies',\n      'gamesStartedPitching', 'runsPitching', 'homeRunsPitching', \n      'strikeOutsPitching', 'baseOnBallsPitching', 'hitsPitching',\n      'inningsPitchedAsFrac', 'earnedRuns', \n      'battersFaced','saves', 'blownSaves', 'pitchingGameScore', \n      'noHitter'\n      ]].\n    sum()\n    ),\n    on = ['dailyDataDate', 'playerId'],\n  how = 'inner'\n  )\n#### Turn games table into 1 row per team-game, then merge with team box scores ####\n# Filter to regular or Postseason games w/ valid scores for this part\ngames_for_stats = games[\n  np.isin(games['gameType'], ['R', 'F', 'D', 'L', 'W', 'C', 'P']) &\n  ~pd.isna(games['homeScore']) &\n  ~pd.isna(games['awayScore'])\n  ]\n# Get games table from home team perspective\ngames_home_perspective = games_for_stats.copy()\n# Change column names so that \"team\" is \"home\", \"opp\" is \"away\"\ngames_home_perspective.columns = [\n  col_value.replace('home', 'team').replace('away', 'opp') for \n    col_value in games_home_perspective.columns.values]\n\ngames_home_perspective['isHomeTeam'] = 1\n\n# Get games table from away team perspective\ngames_away_perspective = games_for_stats.copy()\n# Put together games from home/away perspective to get df w/ 1 row per team game\nteam_games = (pd.concat([\n  games_home_perspective,\n  games_away_perspective\n  ],\n  ignore_index = True)\n  )\n\n# Copy over team box scores data to modify\nteam_game_stats = teamBoxScores.copy()\n\n# Change column names to reflect these are all \"team\" stats - helps \n# to differentiate from individual player stats if/when joining later\nteam_game_stats.columns = [\n  (col_value + 'Team') \n  if (col_value not in ['dailyDataDate', 'home', 'teamId', 'gamePk',\n    'gameDate', 'gameTimeUTC'])\n    else col_value\n  for col_value in team_game_stats.columns.values\n  ]\n\n\n\n  \n              ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #Merge games table with team game stats\nteam_games_with_stats = pd.merge(\n  team_games,\n  team_game_stats.\n    # Drop some fields that are already present in team_games table\n    drop(['home', 'gameDate', 'gameTimeUTC'], axis = 1),\n  on = ['dailyDataDate', 'gamePk', 'teamId'],\n  # Doing this as 'inner' join excludes spring training games, postponed games,\n  # etc. from original games table, but this may be fine for purposes here \n  how = 'inner'\n  )\nteam_date_stats_agg = (team_games_with_stats.\n  groupby(['dailyDataDate', 'teamId', 'gameType', 'oppId', 'oppName'], \n    as_index = False).\n  agg(\n    numGamesTeam = ('gamePk', 'nunique'),\n    winsTeam = ('teamWinner', 'sum'),\n    lossesTeam = ('oppWinner', 'sum'),\n    runsScoredTeam = ('teamScore', 'sum'),\n    runsAllowedTeam = ('oppScore', 'sum')\n    )\n   )\n# Prepare standings table for merge w/ player digital engagement data\n# Pick only certain fields of interest from standings for merge\nstandings_selected_fields = (standings[['dailyDataDate', 'teamId', \n  'streakCode', 'divisionRank', 'leagueRank', 'wildCardRank', 'pct'\n  ]].\n  rename(columns = {'pct': 'winPct'})\n  )\n\n# Change column names to reflect these are all \"team\" standings - helps \n# to differentiate from player-related fields if/when joining later\nstandings_selected_fields.columns = [\n  (col_value + 'Team') \n  if (col_value not in ['dailyDataDate', 'teamId'])\n    else col_value\n  for col_value in standings_selected_fields.columns.values\n  ]\n\nstandings_selected_fields['streakLengthTeam'] = (\n  standings_selected_fields['streakCodeTeam'].\n    str.replace('W', '').\n    str.replace('L', '').\n    astype(float)\n    )\n# Add fields to separate winning and losing streak from streak code\nstandings_selected_fields['winStreakTeam'] = np.where(\n  standings_selected_fields['streakCodeTeam'].str[0] == 'W',\n  standings_selected_fields['streakLengthTeam'],\n  np.nan\n  )\n\nstandings_selected_fields['lossStreakTeam'] = np.where(\n  standings_selected_fields['streakCodeTeam'].str[0] == 'L',\n  standings_selected_fields['streakLengthTeam'],\n  np.nan\n  )\nstandings_for_digital_engagement_merge = (pd.merge(\n  standings_selected_fields,\n  dates_with_info[['dailyDataDate', 'inSeason']],\n  on = ['dailyDataDate'],\n  how = 'left'\n  ).\n  # Limit down standings to only in season version\n  query(\"inSeason\").\n  # Drop fields no longer necessary (in derived values, etc.)\n  drop(['streakCodeTeam', 'streakLengthTeam', 'inSeason'], axis = 1).\n  reset_index(drop = True)\n  )\n#### Merge together various data frames to add date, player, roster, and team info ####\n# Copy over player engagement df to add various pieces to it\nplayer_engagement_with_info = nextDayPlayerEngagement.copy()\n\n# Take \"row mean\" across targets to add (helps with studying all 4 targets at once)\nplayer_engagement_with_info['targetAvg'] = np.mean(\n  player_engagement_with_info[['target1', 'target2', 'target3', 'target4']],\n  axis = 1)\n\n# Merge in date information\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  dates_with_info[['dailyDataDate', 'date', 'year', 'month', 'inSeason',\n    'seasonPart']],\n  on = ['dailyDataDate'],\n  how = 'left'\n  )\n\n# Merge in some player information\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  players[['playerId', 'playerName', 'DOB', 'mlbDebutDate', 'birthCity',\n    'birthStateProvince', 'birthCountry', 'primaryPositionName']],\n   on = ['playerId'],\n   how = 'left'\n   )\n# Merge in some player roster information by date\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  (rosters[['dailyDataDate', 'playerId', 'statusCode', 'status', 'teamId']].\n    rename(columns = {\n      'statusCode': 'rosterStatusCode',\n      'status': 'rosterStatus',\n      'teamId': 'rosterTeamId'\n      })\n    ),\n  on = ['dailyDataDate', 'playerId'],\n  how = 'left'\n  )\n# Merge in team name from player's roster team\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  (teams[['id', 'teamName']].\n    rename(columns = {\n      'id': 'rosterTeamId',\n      'teamName': 'rosterTeamName'\n      })\n    ),\n  on = ['rosterTeamId'],\n  how = 'left'\n  )\n# Merge in some player game stats (previously aggregated) from that date\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  player_date_stats_agg,\n  on = ['dailyDataDate', 'playerId'],\n  how = 'left'\n  )\n# Merge in team name from player's game team\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  (teams[['id', 'teamName']].\n    rename(columns = {\n      'id': 'gameTeamId',\n      'teamName': 'gameTeamName'\n      })\n    ),\n  on = ['gameTeamId'],\n  how = 'left'\n  )\n# Merge in some team game stats/results (previously aggregated) from that date\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  team_date_stats_agg.rename(columns = {'teamId': 'gameTeamId'}),\n  on = ['dailyDataDate', 'gameTeamId'],\n  how = 'left'\n  )\n# Merge in player transactions of note on that date\n    \n# Merge in some pieces of team standings (previously filter/processed) from that date\nplayer_engagement_with_info = pd.merge(\n  player_engagement_with_info,\n  standings_for_digital_engagement_merge.\n    rename(columns = {'teamId': 'gameTeamId'}),\n  on = ['dailyDataDate', 'gameTeamId'],\n  how = 'left'\n  )\ndisplay(player_engagement_with_info)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(player_engagement_with_info.info(max_cols = 200))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### Functions to help with calculating & display grouped target summaries\n\ndef GetGroupedTargetValuesSummary(df_with_group_and_targets, \n  grouping_vars_list):\n    \n    # Group target values by specified variables\n    group_target_values_summary = (df_with_group_and_targets.\n      groupby(grouping_vars_list, as_index = False, dropna = False).\n      agg(\n        numPlayerDates = ('playerId', 'count'),\n        numPlayers = ('playerId', 'nunique'),\n        target1 = ('target1', np.mean),\n        target2 = ('target2', np.mean),\n        target3 = ('target3', np.mean),\n        target4 =  ('target4', np.mean),\n        target1To4Avg = ('targetAvg', np.mean)\n        ).\n      sort_values(['target1To4Avg'], ascending = False,\n        ignore_index = True)\n      )\n    return(group_target_values_summary)\n;\ndef CalcAndDisplayMultipleSetsOfGroupedTargetValues(\n    df_with_group_and_targets, grouping_names_and_vars_sets_df):\n\n    num_group_sets = grouping_names_and_vars_sets_df.shape[0]\n    \n    kaggle_data_tabs = widgets.Tab()\n\n    # Add Output widgets for each group as tabs' children\n    kaggle_data_tabs.children = list([widgets.Output() \n      for group_set in range(0, num_group_sets)])\n\n    # Loop over each group (name/vars combo), create tab, calc/display summary\n    for index, row in grouping_names_and_vars_sets_df.iterrows():\n    \n        this_group_name = row['groupName']\n        this_group_vars_list = row['groupVarsList']\n# Group target values by specified variables\n        group_target_values_summary = GetGroupedTargetValuesSummary(\n          df_with_group_and_targets, this_group_vars_list\n          )\n\n        # Rename tab bar titles to string concatenating grouping vars\n        kaggle_data_tabs.set_title(index, this_group_name)\n\n        # Display corresponding table output for this tab name\n        with kaggle_data_tabs.children[index]:\n            print('Average Daily Digital Engagement by ' + this_group_name)\n            \n            print('Average Target Values Grouped By ' + \n              str(this_group_vars_list))\n\n            display(group_target_values_summary.round(decimals = 1))\n    \n    display(kaggle_data_tabs)\n    ;","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"player_related_grouping_names_and_var_sets = pd.DataFrame(data = {\n  'groupName': [\n    'Player',\n    'Player Team (by Roster)',\n    'Player Roster Status',\n    \n    ],\n  'groupVarsList': pd.Series([\n    ['playerId', 'playerName'], \n    ['rosterTeamName'],\n    ['rosterStatusCode', 'rosterStatus'], \n    \n    ])\n })    \n\nCalcAndDisplayMultipleSetsOfGroupedTargetValues(\n  player_engagement_with_info, \n  player_related_grouping_names_and_var_sets\n  )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Players in the test set. We'll filter our data for only this set of players\npids_test = players.playerId.loc[\n    players.playerForTestSetAndFuturePreds.fillna(False)\n].astype(str)\n\n# Name of target columns\ntargets = [\"target1\", \"target2\", \"target3\", \"target4\"]\n\n\ndef make_playerBoxScores(dfs: dict, features):\n    X = dfs['playerBoxScores'].copy()\n    X = X[['gameDate', 'playerId'] + features]\n    # Set dtypes\n    X = X.astype({name: np.float32 for name in features})\n    X = X.astype({'playerId': str})\n    # Create date index\n    X = X.rename(columns={'gameDate': 'date'})\n    X['date'] = pd.PeriodIndex(X.date, freq='D')\n    # Aggregate multiple games per day by summing\n    X = X.groupby(['date', 'playerId'], as_index=False).sum()\n    return X","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" Y= player_engagement_with_info.iloc[:,[3,4,5,6,7]]  \n Y   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=player_engagement_with_info.iloc[:, np.r_[0:3,8:61]]\nX.tail()\nX.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,X_valid, y_train, y_valid=train_test_split(\n        X,\n        Y,\n        test_size=30,\n        shuffle=False,\n    ) \nX_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader,Dataset\nclass MLBDataset(Dataset):\n    def __init__(self, df, targets=None, mode='train'):\n        self.mode = mode\n        self.data = df\n        if mode == 'train':\n            self.targets = targets\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        if self.mode == 'train':\n            x = self.data[idx]\n            y = np.array(self.targets[idx])\n            return torch.from_numpy(x).float(), torch.from_numpy(y).float()\n        elif self.mode == 'test':\n            return torch.from_numpy(self.data[idx]).float()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader,Dataset\nimport matplotlib.pyplot as plt\n\n\ndevice=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n#hyper parameters\n\ninput_size =140344176#flattening the array of size 28*28\nhidden_size=1024\nnum_epochs = 100\nbatch_size = 32\nval_btch_size=batch_size*8\nlearning_rate = 1e-2\ntargets = ['target1', 'target2', 'target3', 'target4']\n#MNIST\ntrain_dataset= MLBDataset(X_train,y_train)\nval_dataset=MLBDataset(X_valid, y_valid)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle =True)\nval_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=val_btch_size, shuffle=False)\n\n'''examples = iter(train_loader)\nfeatures,targets= examples.next()\nprint(features.shape,targets.shape)\nfor i in range(6):\n    plt.subplot(2,3,i+1)\n    plt.imshow(samples[i][0])\n    #plt.show'''\nclass NeuralNet(nn.Module):\n     def __init__(self,input_size,hidden_size):\n         super(NeuralNet,self).__init__()\n         self.l1= nn.Linear(input_size,hidden_size)\n         self.relu = nn.ReLU()\n         self.l2= nn.Linear(hidden_size,hidden_size)\n         self.l3=nn.Linear(hidden_size,len(targets)) \n     def forward(self,x):\n         out = self.l1(x)\n         out = self.relu(out)\n         out = self.l2(out)\n         out = self.relu(l2)\n         out =slef.l3(out)\n         return out\nmodel= NeuralNet(input_size,hidden_size)\n#loss and optimizer\ncriterion= nn.CrossEntropyLoss()\noptimizer=torch.optim.Adam(model.parameters(),lr=learning_rate)\n\n#training loop\nn_total_steps= len(train_loader)\nfor epoch in range(num_epochs):\n    for i,(features,targets) in enumerate(train_loader):\n        \n        features= features.to(device)\n        targets=labels.to(device)\n        #forward pass\n        outputs=model(features)\n        loss=criterion(features,targets)\n        #backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        if (i+1)%100==0:\n            print(f'epoch{epoch+1}/{num_epochs},step {i+1}/{n_total_steps},loss={loss.item():.4f}')\n#Testing\nwith torch.no_grad():\n    n_correct=0\n    n_samples=0\n    for images,labels in test_loader:\n        images=images.reshape(-1,28*28).to(device)\n        labels=labels.to(device)\n        outputs=model(images)\n    \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}