{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a id=\"top\"></a>\n\n<h2 class=\"list-group-item list-group-item-action active\" data-toggle=\"list\" role=\"tab\" aria-controls=\"home\">MLB Player Digital Engagement Forecasting</h2>\n\n<img src='https://media.istockphoto.com/vectors/several-baseball-players-in-different-positions-vector-id134085440?k=6&m=134085440&s=612x612&w=0&h=5_Qznbnajho0p04xvUVcVkb9vLPAs_TlGspOqnkI-d8='></img>\n\n* [0. Players](#0)\n* [1. Seasons](#1)\n* [2. Teams](#2)\n* [3. Train](#3)\n    * [3.1 nextDayPlayerEngagement](#3.1)\n        * [3.1.1 PlayerId[628317] Example](#3.1.1)\n        * [3.1.2 Targets Vs primaryPosition](#3.1.2)\n        * [ 3.1.3 Targets Vs BMI, Heaight, Weight, Age](#3.1.3)\n    * [3.2 rosters](3.2)\n        * [3.2.1 isActive Feature](#3.2.1)\n        * [3.2.2 Illness Feature](#3.2.2)\n        * [3.2.3 Bereavement Feature](#3.2.3)\n        * [3.2.4 Deceased Feature](#3.2.4)\n        * [3.2.5 Family Medical Emergency Feature](#3.2.5)\n        * [3.2.6 Paternity & Paternity List Feature](#3.2.6)\n        * [3.2.7 Reassigned to Major Features](#3.2.7)\n        * [3.2.8 Reassigned  to Minor Features](#3.2.8)\n        * [3.2.9 Reserve List (Minors) Features](#3.2.9)\n        * [3.2.10 Suspended Features](#3.2.10)","metadata":{}},{"cell_type":"code","source":"!pip install -q jupyter-dash\n\nimport os\nimport json\nimport datetime\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport dash_core_components as dcc\nimport dash_html_components as html\n\n\nfrom jupyter_dash import JupyterDash\nfrom dash.dependencies import Input, Output\nfrom dateutil import parser\nfrom tqdm.notebook import tqdm","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:33.257702Z","iopub.execute_input":"2021-06-15T20:21:33.258570Z","iopub.status.idle":"2021-06-15T20:21:45.167155Z","shell.execute_reply.started":"2021-06-15T20:21:33.258353Z","shell.execute_reply":"2021-06-15T20:21:45.165918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"0\"></a>\n## 0. Players\n\n<img src='https://upload.wikimedia.org/wikipedia/commons/thumb/8/88/Baseball_positions.svg/1200px-Baseball_positions.svg.png'></img>\n\n* `playerId` - Unique identifier for a player.\n* `playerName`\n* `DOB` - Player’s date of birth.\n* `mlbDebutDate`\n* `birthCity`\n* `birthStateProvince`\n* `birthCountry`\n* `heightInches`\n* `weight`\n* `primaryPositionCode` - Player’s primary position code, details are [here](https://statsapi.mlb.com/api/v1/positions).\n* `primaryPositionName` - player’s primary position, details are [here](https://statsapi.mlb.com/api/v1/positions).\n* `playerForTestSetAndFuturePreds` - Boolean, true if player is among those for whom predictions are to be made in test data","metadata":{}},{"cell_type":"code","source":"players_df = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/players.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:45.168951Z","iopub.execute_input":"2021-06-15T20:21:45.169256Z","iopub.status.idle":"2021-06-15T20:21:45.195527Z","shell.execute_reply.started":"2021-06-15T20:21:45.169226Z","shell.execute_reply":"2021-06-15T20:21:45.194624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:21:45.197163Z","iopub.execute_input":"2021-06-15T20:21:45.197443Z","iopub.status.idle":"2021-06-15T20:21:45.224306Z","shell.execute_reply.started":"2021-06-15T20:21:45.197417Z","shell.execute_reply":"2021-06-15T20:21:45.223299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players_df['densety'] = 1 / len(players_df)\nplayers_df = players_df.fillna('NaN')\n\nfig = px.sunburst(players_df, path=['playerForTestSetAndFuturePreds', 'primaryPositionName', 'birthCountry', 'birthStateProvince', 'birthCity'], values='densety')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:45.225924Z","iopub.execute_input":"2021-06-15T20:21:45.226295Z","iopub.status.idle":"2021-06-15T20:21:47.079841Z","shell.execute_reply.started":"2021-06-15T20:21:45.226260Z","shell.execute_reply":"2021-06-15T20:21:47.078999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So about 57.5 % of players is used on test stage. Most players is Pitchers from California USA.","metadata":{}},{"cell_type":"code","source":"_players_df = players_df.loc[players_df.playerForTestSetAndFuturePreds != 'NaN', :]\n_players_df[\"heightInches / 100\"] = _players_df[\"heightInches\"] / 100\n\nfig = px.violin(_players_df, y=\"heightInches / 100\", x=\"playerForTestSetAndFuturePreds\", color=\"primaryPositionName\", box=False)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.080902Z","iopub.execute_input":"2021-06-15T20:21:47.081496Z","iopub.status.idle":"2021-06-15T20:21:47.249780Z","shell.execute_reply.started":"2021-06-15T20:21:47.081461Z","shell.execute_reply":"2021-06-15T20:21:47.248984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The tallest players in positions:\n* outfilder\n* first base\n* pitcher","metadata":{}},{"cell_type":"code","source":"_players_df[\"weight / 100\"] = _players_df[\"weight\"] / 100\nfig = px.violin(_players_df, y=\"weight / 100\", x=\"playerForTestSetAndFuturePreds\", color=\"primaryPositionName\", box=False,)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.250777Z","iopub.execute_input":"2021-06-15T20:21:47.251151Z","iopub.status.idle":"2021-06-15T20:21:47.373092Z","shell.execute_reply.started":"2021-06-15T20:21:47.251123Z","shell.execute_reply":"2021-06-15T20:21:47.372229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Players with biggest weights in positions:\n* catcher\n* first base\n* pitcher","metadata":{}},{"cell_type":"markdown","source":"Let's calculate [BMI](https://en.wikipedia.org/wiki/Body_mass_index):\n$$ BMI = \\frac{weight}{height^2} * 703 $$","metadata":{}},{"cell_type":"code","source":"_players_df[\"BMI\"] = _players_df[\"weight\"] / (_players_df[\"weight\"] * _players_df[\"weight\"]) * 703\nfig = px.violin(_players_df, y=\"BMI\", x=\"playerForTestSetAndFuturePreds\", color=\"primaryPositionName\", box=False,)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.374147Z","iopub.execute_input":"2021-06-15T20:21:47.374539Z","iopub.status.idle":"2021-06-15T20:21:47.495957Z","shell.execute_reply.started":"2021-06-15T20:21:47.374511Z","shell.execute_reply":"2021-06-15T20:21:47.495154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Players with less BMI in positions:\n* Designated Hitter\n* First Base\n\nPlayers with big BMI in positions:\n* Shortstop","metadata":{}},{"cell_type":"code","source":"def calculateAge(birthDate):\n    birthDate = parser.parse(birthDate)\n    today = datetime.date.today()\n    age = (today.year - birthDate.year - \n         ((today.month, today.day) <\n         (birthDate.month, birthDate.day)))\n    return age","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.498059Z","iopub.execute_input":"2021-06-15T20:21:47.498475Z","iopub.status.idle":"2021-06-15T20:21:47.504114Z","shell.execute_reply.started":"2021-06-15T20:21:47.498445Z","shell.execute_reply":"2021-06-15T20:21:47.503113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_players_df['age'] = _players_df['DOB'].apply(lambda x: calculateAge(x))\n_players_df['age / 100'] = _players_df['age']/100\nfig = px.violin(_players_df, y=\"age / 100\", x=\"playerForTestSetAndFuturePreds\", color=\"primaryPositionName\", box=False, points='all')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.505802Z","iopub.execute_input":"2021-06-15T20:21:47.506126Z","iopub.status.idle":"2021-06-15T20:21:47.752186Z","shell.execute_reply.started":"2021-06-15T20:21:47.506089Z","shell.execute_reply":"2021-06-15T20:21:47.751176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like the test group players are younger. Let's calculate average age for two groups.","metadata":{}},{"cell_type":"code","source":"print('Average age players in test group:', _players_df.loc[_players_df.playerForTestSetAndFuturePreds == True, 'age'].mean())\nprint('Average age players in train only group:', _players_df.loc[_players_df.playerForTestSetAndFuturePreds == False, 'age'].mean())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.753330Z","iopub.execute_input":"2021-06-15T20:21:47.753603Z","iopub.status.idle":"2021-06-15T20:21:47.761512Z","shell.execute_reply.started":"2021-06-15T20:21:47.753578Z","shell.execute_reply":"2021-06-15T20:21:47.760750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(_players_df, x=\"age\", y=\"BMI\", color=\"playerForTestSetAndFuturePreds\", opacity=0.25, trendline='ols')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:47.762617Z","iopub.execute_input":"2021-06-15T20:21:47.762993Z","iopub.status.idle":"2021-06-15T20:21:48.516564Z","shell.execute_reply.started":"2021-06-15T20:21:47.762961Z","shell.execute_reply":"2021-06-15T20:21:48.515660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Players BMI with age decrease.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 1. Seasons\n* `seasonId`\n* `seasonStartDate`\n* `seasonEndDate`\n* `preSeasonStartDate`\n* `preSeasonEndDate`\n* `regularSeasonStartDate`\n* `regularSeasonEndDate`\n* `lastDate1stHalf`\n* `allStarDate`\n* `firstDate2ndHalf`\n* `postSeasonStartDate`\n* `postSeasonEndDate`","metadata":{}},{"cell_type":"code","source":"seasons_df = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/seasons.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.518016Z","iopub.execute_input":"2021-06-15T20:21:48.518516Z","iopub.status.idle":"2021-06-15T20:21:48.529153Z","shell.execute_reply.started":"2021-06-15T20:21:48.518483Z","shell.execute_reply":"2021-06-15T20:21:48.528284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seasons_df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.530459Z","iopub.execute_input":"2021-06-15T20:21:48.530981Z","iopub.status.idle":"2021-06-15T20:21:48.551111Z","shell.execute_reply.started":"2021-06-15T20:21:48.530949Z","shell.execute_reply":"2021-06-15T20:21:48.549595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = []\nend = []\nttype = []\n\nfor i, row in seasons_df.iterrows():\n    start.append(row.seasonStartDate)\n    end.append(row.seasonEndDate)\n    ttype.append('Season')\n    \n    start.append(row.preSeasonStartDate)\n    end.append(row.preSeasonEndDate)\n    ttype.append('Pre Season')   \n    \n    start.append(row.regularSeasonStartDate)\n    end.append(row.regularSeasonEndDate)\n    ttype.append('Regular Season')\n    \n    start.append(row.postSeasonStartDate)\n    end.append(row.postSeasonEndDate)\n    ttype.append('Post Season')\n    \nseson_df_timeline = pd.DataFrame({'Start': start, 'End': end, 'Type': ttype})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.552329Z","iopub.execute_input":"2021-06-15T20:21:48.552906Z","iopub.status.idle":"2021-06-15T20:21:48.566124Z","shell.execute_reply.started":"2021-06-15T20:21:48.552863Z","shell.execute_reply":"2021-06-15T20:21:48.565332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.timeline(seson_df_timeline, x_start=\"Start\", x_end=\"End\", y=\"Type\", color=\"Type\")\n\nfor i, row in seasons_df.iterrows():\n    fig.add_shape(type='line',\n                yref=\"y\",\n                xref=\"x\",\n                x0=row.lastDate1stHalf,\n                x1=row.lastDate1stHalf,\n                y0=-1,\n                y1=4,\n                line=dict(color='green', width=1))\n    fig.add_annotation(\n                x=row.lastDate1stHalf,\n                y=1.06,\n                yref='paper',\n                showarrow=False,\n                text=f'lastDate1stHalf {row.seasonId}')\n    \n    fig.add_shape(type='line',\n                yref=\"y\",\n                xref=\"x\",\n                x0=row.firstDate2ndHalf,\n                x1=row.firstDate2ndHalf,\n                y0=-1,\n                y1=4,\n                line=dict(color='red', width=1))\n    fig.add_annotation(\n                x=row.firstDate2ndHalf,\n                y=-0.12,\n                yref='paper',\n                showarrow=False,\n                text=f'firstDate2ndHalf {row.seasonId}')\n    \n    if isinstance(row.allStarDate, str):\n        fig.add_shape(type='line',\n                yref=\"y\",\n                xref=\"x\",\n                x0=row.allStarDate,\n                x1=row.allStarDate,\n                y0=-1,\n                y1=4,\n                line=dict(color='blue', width=1))\n        fig.add_annotation(\n                x=row.allStarDate,\n                y=1.10,\n                yref='paper',\n                showarrow=False,\n                textangle = 0, \n                text=f'allStarDate {row.seasonId}')\nfig.update_yaxes(autorange=\"reversed\")\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.567346Z","iopub.execute_input":"2021-06-15T20:21:48.567953Z","iopub.status.idle":"2021-06-15T20:21:48.783096Z","shell.execute_reply.started":"2021-06-15T20:21:48.567909Z","shell.execute_reply":"2021-06-15T20:21:48.781998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n## 2. Teams\n* `id` - teamId\n* `name`\n* `teamName`\n* `teamCode`\n* `shortName`\n* `abbreviation`\n* `locationName`\n* `leagueId`\n* `leagueName`\n* `divisionId`\n* `divisionName`\n* `venueId`\n* `venueName`","metadata":{}},{"cell_type":"code","source":"teams_df = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/teams.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.784527Z","iopub.execute_input":"2021-06-15T20:21:48.784982Z","iopub.status.idle":"2021-06-15T20:21:48.797386Z","shell.execute_reply.started":"2021-06-15T20:21:48.784945Z","shell.execute_reply":"2021-06-15T20:21:48.796306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"teams_df.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.798868Z","iopub.execute_input":"2021-06-15T20:21:48.799281Z","iopub.status.idle":"2021-06-15T20:21:48.818247Z","shell.execute_reply.started":"2021-06-15T20:21:48.799231Z","shell.execute_reply":"2021-06-15T20:21:48.817268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"teams_df['densety'] = 1 / len(teams_df)\nteams_df = teams_df.fillna('NaN')\n\nfig = px.sunburst(teams_df, path=['leagueName', 'divisionName', 'name'], values='densety')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.819737Z","iopub.execute_input":"2021-06-15T20:21:48.820301Z","iopub.status.idle":"2021-06-15T20:21:48.942534Z","shell.execute_reply.started":"2021-06-15T20:21:48.820257Z","shell.execute_reply":"2021-06-15T20:21:48.941531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3\"></a>\n## 3. Train\n\nThis contains data on MLB players active at some point since 2018. Predictions are only scored for those players active in 2021 (see above), but previous seasons’ players are included here to provide more data for exploration and modeling purposes.\n\n^indicates that a more complete walkthrough is below\n\n   * `date` - Integer formatted date, which is a primary index of the CSV.\n   * `nextDayPlayerEngagement^` - Nested JSON containing all modeling targets from the following day.\n   * `games^` - Nested JSON containing all game information for a given day. Includes spring training and exhibition games along with regular season, Postseason, and All-Star games.\n   * `rosters^` - Nested JSON containing all roster information for a given day. Includes in-season and offseason team rosters.\n   * `playerBoxScores^` - Nested JSON containing game stats aggregated at the player game level for a given day. Includes regular season, Postseason, and All-Star games.\n   * `teamBoxScores^` - Nested JSON containing game stats aggregated at the team game level for a given day. Includes regular season, Postseason, and All-Star games.\n   * `transactions^` - Nested JSON containing all transaction information involving MLB teams for a given day.\n   * `standings^` - Nested JSON containing all standings information involving MLB teams for a given day.\n   * `awards^` - Nested JSON containing all awards or honors handed out on a given day.\n   * `events^` - Nested JSON containing all on-field game events for a given day. Includes regular season and Postseason games.\n   * `playerTwitterFollowers^` - Nested JSON containing some players’ number of Twitter followers on that day.\n   * `teamTwitterFollowers^` - Nested JSON containing each team’s number of Twitter followers on that day.\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:21:48.944023Z","iopub.execute_input":"2021-06-15T20:21:48.944586Z","iopub.status.idle":"2021-06-15T20:23:10.944794Z","shell.execute_reply.started":"2021-06-15T20:21:48.944542Z","shell.execute_reply":"2021-06-15T20:23:10.943621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:23:10.946601Z","iopub.execute_input":"2021-06-15T20:23:10.947037Z","iopub.status.idle":"2021-06-15T20:23:10.979767Z","shell.execute_reply.started":"2021-06-15T20:23:10.946991Z","shell.execute_reply":"2021-06-15T20:23:10.978771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3.1\"></a>\n## 3.1 nextDayPlayerEngagement\n   * `engagementMetricsDate` - date of player engagement metrics, based on US Pacific Time (aligns with previous day’s games, rosters, on-field statistics, transactions, awards, etc.).\n   * `playerId`\n   * `target1`\n   * `target2`\n   * `target3`\n   * `target4`\n\ntarget1-target4 are each daily indexes of digital engagement on a 0-100 scale.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.1.1\"></a>\n### 3.1.1 PlayerId[628317] Example","metadata":{}},{"cell_type":"code","source":"records = []\nfor nextDayPlayerEngagement in train_df.nextDayPlayerEngagement.values:\n    records.extend(filter(lambda x:  x['playerId'] == 628317, json.loads(nextDayPlayerEngagement)))\nplayerTarget = pd.DataFrame.from_records(records)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:23:10.981233Z","iopub.execute_input":"2021-06-15T20:23:10.981793Z","iopub.status.idle":"2021-06-15T20:23:20.139356Z","shell.execute_reply.started":"2021-06-15T20:23:10.981749Z","shell.execute_reply":"2021-06-15T20:23:20.138597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(playerTarget, x=\"engagementMetricsDate\", y=[\"target1\", \"target2\", \"target3\", \"target4\"])\nfig.update_layout(\n    title={\n        'text': \"PlayerId: 628317\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name in ['target2', 'target3', 'target4'] else ())\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:23:20.140387Z","iopub.execute_input":"2021-06-15T20:23:20.140789Z","iopub.status.idle":"2021-06-15T20:23:20.291680Z","shell.execute_reply.started":"2021-06-15T20:23:20.140760Z","shell.execute_reply":"2021-06-15T20:23:20.290665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3.1.2\"></a>\n### 3.1.2 Targets Vs primaryPosition","metadata":{}},{"cell_type":"code","source":"records = []\nfor pN in tqdm(players_df.primaryPositionName.unique()):\n    pids = players_df.loc[players_df.primaryPositionName==pN, 'playerId'].values\n    for nextDayPlayerEngagement in tqdm(train_df.nextDayPlayerEngagement.values, total=len(train_df.nextDayPlayerEngagement.values)):\n        filtered = list(filter(lambda x:  x['playerId'] in pids, json.loads(nextDayPlayerEngagement)))\n        records.extend([\n            {\n                'engagementMetricsDate': filtered[0]['engagementMetricsDate'],\n                'target1': np.mean([f['target1'] for f in filtered]),\n                'target2': np.mean([f['target2'] for f in filtered]),\n                'target3': np.mean([f['target3'] for f in filtered]),\n                'target4': np.mean([f['target4'] for f in filtered]),\n                'primaryPositionName': pN\n            }\n        ])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:23:20.296149Z","iopub.execute_input":"2021-06-15T20:23:20.296486Z","iopub.status.idle":"2021-06-15T20:27:20.384311Z","shell.execute_reply.started":"2021-06-15T20:23:20.296455Z","shell.execute_reply":"2021-06-15T20:27:20.383263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targetStatByPosition = pd.DataFrame.from_records(records)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:20.386933Z","iopub.execute_input":"2021-06-15T20:27:20.387233Z","iopub.status.idle":"2021-06-15T20:27:20.427437Z","shell.execute_reply.started":"2021-06-15T20:27:20.387195Z","shell.execute_reply":"2021-06-15T20:27:20.426281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(targetStatByPosition, x=\"engagementMetricsDate\", y=\"target1\", color='primaryPositionName')\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['Pitcher'] else ())\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:20.428967Z","iopub.execute_input":"2021-06-15T20:27:20.429386Z","iopub.status.idle":"2021-06-15T20:27:20.685853Z","shell.execute_reply.started":"2021-06-15T20:27:20.429341Z","shell.execute_reply":"2021-06-15T20:27:20.684730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.violin(targetStatByPosition, y=\"target1\", x=\"primaryPositionName\", color=\"primaryPositionName\", box=False)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:20.687527Z","iopub.execute_input":"2021-06-15T20:27:20.687895Z","iopub.status.idle":"2021-06-15T20:27:21.084326Z","shell.execute_reply.started":"2021-06-15T20:27:20.687865Z","shell.execute_reply":"2021-06-15T20:27:21.083298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(targetStatByPosition, x=\"engagementMetricsDate\", y=\"target2\", color='primaryPositionName')\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['Pitcher'] else ())\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:21.085731Z","iopub.execute_input":"2021-06-15T20:27:21.086056Z","iopub.status.idle":"2021-06-15T20:27:21.334745Z","shell.execute_reply.started":"2021-06-15T20:27:21.086025Z","shell.execute_reply":"2021-06-15T20:27:21.333488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.violin(targetStatByPosition, y=\"target2\", x=\"primaryPositionName\", color=\"primaryPositionName\", box=False)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:21.336176Z","iopub.execute_input":"2021-06-15T20:27:21.336459Z","iopub.status.idle":"2021-06-15T20:27:21.585298Z","shell.execute_reply.started":"2021-06-15T20:27:21.336433Z","shell.execute_reply":"2021-06-15T20:27:21.584283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(targetStatByPosition, x=\"engagementMetricsDate\", y=\"target3\", color='primaryPositionName')\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['Pitcher'] else ())\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:21.586740Z","iopub.execute_input":"2021-06-15T20:27:21.587051Z","iopub.status.idle":"2021-06-15T20:27:21.833291Z","shell.execute_reply.started":"2021-06-15T20:27:21.587020Z","shell.execute_reply":"2021-06-15T20:27:21.832237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.violin(targetStatByPosition, y=\"target3\", x=\"primaryPositionName\", color=\"primaryPositionName\", box=False)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:21.834942Z","iopub.execute_input":"2021-06-15T20:27:21.835434Z","iopub.status.idle":"2021-06-15T20:27:22.061797Z","shell.execute_reply.started":"2021-06-15T20:27:21.835392Z","shell.execute_reply":"2021-06-15T20:27:22.060752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(targetStatByPosition, x=\"engagementMetricsDate\", y=\"target4\", color='primaryPositionName')\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['Pitcher'] else ())\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:22.063142Z","iopub.execute_input":"2021-06-15T20:27:22.063454Z","iopub.status.idle":"2021-06-15T20:27:22.309305Z","shell.execute_reply.started":"2021-06-15T20:27:22.063427Z","shell.execute_reply":"2021-06-15T20:27:22.307986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.violin(targetStatByPosition, y=\"target4\", x=\"primaryPositionName\", color=\"primaryPositionName\", box=False)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:22.310821Z","iopub.execute_input":"2021-06-15T20:27:22.311129Z","iopub.status.idle":"2021-06-15T20:27:22.543371Z","shell.execute_reply.started":"2021-06-15T20:27:22.311101Z","shell.execute_reply":"2021-06-15T20:27:22.542361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So `primaryPositionName` is categorical feature which have none linear relation with targets value distributions.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.1.3\"></a>\n### 3.1.3 Targets Vs BMI, Heaight, Weight, Age","metadata":{}},{"cell_type":"code","source":"playerStat = {}\nfor nextDayPlayerEngagement in tqdm(train_df.nextDayPlayerEngagement.values, total=len(train_df.nextDayPlayerEngagement.values)):\n    nextDayPlayerEngagement = json.loads(nextDayPlayerEngagement)\n    for player in nextDayPlayerEngagement:\n        if player['playerId'] in playerStat:\n            playerStat[player['playerId']] += np.array([\n                    float(player['target1']), float(player['target2']), float(player['target3']), float(player['target4']),\n                    1., 1., 1., 1.\n            ])\n        else:\n            playerStat[player['playerId']] = np.array([\n                    float(player['target1']), float(player['target2']), float(player['target3']), float(player['target4']),\n                    1., 1., 1., 1.\n            ])\n            \nfor i in range(1, 5):\n    _players_df[f'target{i}Mean'] = 0\n    \nfor pid, v in playerStat.items():\n    _players_df.loc[players_df.playerId == pid, ['target1Mean', 'target2Mean', 'target3Mean', 'target4Mean']] = np.array([\n        v[0 + i]/v[4 + i] for i in range(4)\n    ])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:22.544665Z","iopub.execute_input":"2021-06-15T20:27:22.544951Z","iopub.status.idle":"2021-06-15T20:27:55.463788Z","shell.execute_reply.started":"2021-06-15T20:27:22.544923Z","shell.execute_reply":"2021-06-15T20:27:55.462774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(_players_df, x=\"BMI\", y=['target1Mean', 'target2Mean', 'target3Mean', 'target4Mean'], opacity=0.25, trendline='ols')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:55.465058Z","iopub.execute_input":"2021-06-15T20:27:55.465356Z","iopub.status.idle":"2021-06-15T20:27:55.653775Z","shell.execute_reply.started":"2021-06-15T20:27:55.465327Z","shell.execute_reply":"2021-06-15T20:27:55.652803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(_players_df, x=\"heightInches\", y=['target1Mean', 'target2Mean', 'target3Mean', 'target4Mean'], opacity=0.25, trendline='ols')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:55.655089Z","iopub.execute_input":"2021-06-15T20:27:55.655376Z","iopub.status.idle":"2021-06-15T20:27:55.829463Z","shell.execute_reply.started":"2021-06-15T20:27:55.655346Z","shell.execute_reply":"2021-06-15T20:27:55.828497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(_players_df, x=\"weight\", y=['target1Mean', 'target2Mean', 'target3Mean', 'target4Mean'], opacity=0.25, trendline='ols')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:55.830982Z","iopub.execute_input":"2021-06-15T20:27:55.831575Z","iopub.status.idle":"2021-06-15T20:27:56.168361Z","shell.execute_reply.started":"2021-06-15T20:27:55.831529Z","shell.execute_reply":"2021-06-15T20:27:56.167348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(_players_df, x=\"age\", y=['target1Mean', 'target2Mean', 'target3Mean', 'target4Mean'], opacity=0.25, trendline='ols')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:56.169670Z","iopub.execute_input":"2021-06-15T20:27:56.169950Z","iopub.status.idle":"2021-06-15T20:27:56.342797Z","shell.execute_reply.started":"2021-06-15T20:27:56.169922Z","shell.execute_reply":"2021-06-15T20:27:56.341765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Targets have increased trend with age, weight, height & decreased trend for BMI index.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2\"></a>\n## 3.2 rosters\n   * `playerId` - Unique identifier for a player.\n   * `gameDate`\n   * `teamId` - teamId that player is on that date.\n   * `statusCode` - Roster status abbreviation.\n   * `status` - Descriptive roster status.\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T19:31:38.328106Z","iopub.execute_input":"2021-06-14T19:31:38.32854Z","iopub.status.idle":"2021-06-14T19:31:38.336709Z","shell.execute_reply.started":"2021-06-14T19:31:38.328505Z","shell.execute_reply":"2021-06-14T19:31:38.335077Z"}}},{"cell_type":"markdown","source":"<a id=\"3.2.1\"></a>\n### 3.2.1 isActive Feature","metadata":{}},{"cell_type":"code","source":"playerActivity = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nplayerTarget = {pid: np.zeros((len(train_df), 4)) for pid in players_df.playerId}\n\nfor i, nextDayPlayerEngagement in tqdm(enumerate(train_df.nextDayPlayerEngagement.values)):\n    nextDayPlayerEngagement = json.loads(nextDayPlayerEngagement)\n    for ndpe in nextDayPlayerEngagement:\n        if ndpe['playerId'] in playerTarget:\n            playerTarget[ndpe['playerId']][i] = [ndpe[f'target{j}'] for j in range(1, 5)]\n\nstatus = set()\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    for r in json.loads(roster):\n        if r['playerId'] in playerActivity:\n            status.add(r['status'])\n            playerActivity[r['playerId']][i] += int(r['status'] == 'Active')\ntrain_df.date = train_df.date.apply(lambda x: parser.parse(str(x)))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:27:56.344050Z","iopub.execute_input":"2021-06-15T20:27:56.344348Z","iopub.status.idle":"2021-06-15T20:28:27.351329Z","shell.execute_reply.started":"2021-06-15T20:27:56.344318Z","shell.execute_reply":"2021-06-15T20:28:27.350578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 628317\n_ex = pd.DataFrame({'date': train_df.date, 'Activity': playerActivity[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['Activity', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['Activity', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': \"PlayerId: 628317\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:27.352343Z","iopub.execute_input":"2021-06-15T20:28:27.352782Z","iopub.status.idle":"2021-06-15T20:28:27.699936Z","shell.execute_reply.started":"2021-06-15T20:28:27.352751Z","shell.execute_reply":"2021-06-15T20:28:27.698875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onActive = np.zeros((4,))\nonInActive = np.zeros((4,))\nonActiveCount = 0\nonInActiveCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    activity = playerActivity[pid]\n    \n    active = target[activity > 0]\n    inactive = target[activity == 0]\n        \n    onActiveCount += 1 if len(active) > 0 else 0\n    onInActiveCount += 1 if len(inactive) > 0 else 0\n    for j in range(0, 4):\n        onActive[j] += np.mean(active[:, j]) if len(active) > 0 else 0\n        onInActive[j] += np.mean(inactive[:, j]) if len(inactive) > 0 else 0\nonActive /= onActiveCount\nonInActive /= onInActiveCount","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:27.701277Z","iopub.execute_input":"2021-06-15T20:28:27.701582Z","iopub.status.idle":"2021-06-15T20:28:28.029034Z","shell.execute_reply.started":"2021-06-15T20:28:27.701551Z","shell.execute_reply":"2021-06-15T20:28:28.028149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ex = pd.DataFrame({'isActive': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onActive[0], onInActive[0], onActive[1], onInActive[1], onActive[2], onInActive[2], onActive[3], onInActive[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isActive', barmode='group')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:28.030519Z","iopub.execute_input":"2021-06-15T20:28:28.031024Z","iopub.status.idle":"2021-06-15T20:28:28.110387Z","shell.execute_reply.started":"2021-06-15T20:28:28.030975Z","shell.execute_reply":"2021-06-15T20:28:28.109572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So for player which play on game targets value (`target1`, `target2`, `target4`) is greater then for inactive.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2.2\"></a>\n### 3.2.2 Illness Feature","metadata":{}},{"cell_type":"code","source":"playerIL = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nlastDate = None\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    updatedPid = set()\n    for r in json.loads(roster):\n        if r['playerId'] in playerIL:\n            if r['status'] == '10-day IL':\n                playerIL[r['playerId']][i] += 10\n                updatedPid.add(r['playerId'])\n            elif r['status'] == '60-day IL':\n                playerIL[r['playerId']][i] += 60\n                updatedPid.add(r['playerId'])\n                \n    if lastDate is not None:\n        day = (train_df.date[i] - lastDate).days\n        for pid in playerIL:\n            if pid not in updatedPid:\n                playerIL[pid][i] += max(0, playerIL[pid][i-1] - 1)\n    lastDate = train_df.date[i]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:28.111512Z","iopub.execute_input":"2021-06-15T20:28:28.111950Z","iopub.status.idle":"2021-06-15T20:28:40.902466Z","shell.execute_reply.started":"2021-06-15T20:28:28.111904Z","shell.execute_reply":"2021-06-15T20:28:40.901551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 622554\n_ex = pd.DataFrame({'date': train_df.date, 'IL Days': playerIL[pid], \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['IL Days', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['IL Days', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:40.903725Z","iopub.execute_input":"2021-06-15T20:28:40.904029Z","iopub.status.idle":"2021-06-15T20:28:41.239898Z","shell.execute_reply.started":"2021-06-15T20:28:40.903999Z","shell.execute_reply":"2021-06-15T20:28:41.238723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"illness = np.concatenate([v for v in playerIL.values()], axis=0)\ntargets = np.concatenate([v[:, 0] for v in playerTarget.values()], axis=0)\nbins = np.array(list(range(0, 100)))\nvalue = np.array([np.sum(illness[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(illness)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target1', 'y':'illness'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:41.241194Z","iopub.execute_input":"2021-06-15T20:28:41.241470Z","iopub.status.idle":"2021-06-15T20:28:41.823227Z","shell.execute_reply.started":"2021-06-15T20:28:41.241443Z","shell.execute_reply":"2021-06-15T20:28:41.822229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = np.concatenate([v[:, 1] for v in playerTarget.values()], axis=0)\nvalue = np.array([np.sum(illness[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(illness)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target2', 'y':'illness'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:41.824786Z","iopub.execute_input":"2021-06-15T20:28:41.825213Z","iopub.status.idle":"2021-06-15T20:28:42.466386Z","shell.execute_reply.started":"2021-06-15T20:28:41.825163Z","shell.execute_reply":"2021-06-15T20:28:42.465238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = np.concatenate([v[:, 2] for v in playerTarget.values()], axis=0)\nvalue = np.array([np.sum(illness[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(illness)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target3', 'y':'illness'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:42.467589Z","iopub.execute_input":"2021-06-15T20:28:42.467918Z","iopub.status.idle":"2021-06-15T20:28:43.106915Z","shell.execute_reply.started":"2021-06-15T20:28:42.467888Z","shell.execute_reply":"2021-06-15T20:28:43.105781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = np.concatenate([v[:, 3] for v in playerTarget.values()], axis=0)\nvalue = np.array([np.sum(illness[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(illness)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target4', 'y':'illness'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:43.108366Z","iopub.execute_input":"2021-06-15T20:28:43.108668Z","iopub.status.idle":"2021-06-15T20:28:43.717042Z","shell.execute_reply.started":"2021-06-15T20:28:43.108627Z","shell.execute_reply":"2021-06-15T20:28:43.715984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This means that the sick player has low KPI values, which is logical. ","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2.3\"></a>\n### 3.2.3 Bereavement Feature","metadata":{}},{"cell_type":"code","source":"playerBereavement = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    for r in json.loads(roster):\n        if r['playerId'] in playerBereavement:\n            playerBereavement[r['playerId']][i] += int(r['status'] == 'Bereavement List')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:43.718499Z","iopub.execute_input":"2021-06-15T20:28:43.718915Z","iopub.status.idle":"2021-06-15T20:28:49.335292Z","shell.execute_reply.started":"2021-06-15T20:28:43.718884Z","shell.execute_reply":"2021-06-15T20:28:49.334354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onBereavement = np.zeros((4,))\nonUnbereavement = np.zeros((4,))\nonBereavementCount = 0\nonUnBereavementCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    bereavement = playerBereavement[pid]\n    \n    tunbereavement = target[bereavement == 0]\n    tbereavement = target[bereavement > 0]\n    \n    onBereavementCount += 1 if len(tbereavement) > 0 else 0\n    onUnBereavementCount += 1 if len(tunbereavement) > 0 else 0\n    for j in range(0, 4):\n        onBereavement[j] += np.mean(tbereavement[:, j]) if len(tbereavement) > 0 else 0\n        onUnbereavement[j] += np.mean(tunbereavement[:, j]) if len(tunbereavement) > 0 else 0\nonBereavement /= onBereavementCount\nonUnbereavement /= onUnBereavementCount\n\n_ex = pd.DataFrame({'isBereavement': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onBereavement[0], onUnbereavement[0], onBereavement[1], onUnbereavement[1], onBereavement[2], onUnbereavement[2], onBereavement[3], onUnbereavement[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isBereavement', barmode='group')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:49.336521Z","iopub.execute_input":"2021-06-15T20:28:49.337118Z","iopub.status.idle":"2021-06-15T20:28:49.659518Z","shell.execute_reply.started":"2021-06-15T20:28:49.337082Z","shell.execute_reply":"2021-06-15T20:28:49.658670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hm... Some targets value for a player with bereavement greater than for a player without bereavement.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2.4\"></a>\n### 3.2.4 Deceased Feature","metadata":{}},{"cell_type":"code","source":"playerDeceased = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    for r in json.loads(roster):\n        if r['playerId'] in playerDeceased:\n            playerDeceased[r['playerId']][i:] += int(r['status'] == 'Deceased')\n            if r['status'] == 'Deceased':\n                print('Deceased Player:', r['playerId'], players_df.loc[players_df.playerId == pid, 'playerName'].values[0])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:28:49.660877Z","iopub.execute_input":"2021-06-15T20:28:49.661301Z","iopub.status.idle":"2021-06-15T20:29:00.528070Z","shell.execute_reply.started":"2021-06-15T20:28:49.661271Z","shell.execute_reply":"2021-06-15T20:29:00.526336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 572140\n_ex = pd.DataFrame({'date': train_df.date, 'isDeceased': playerDeceased[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isDeceased', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isDeceased', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:00.530031Z","iopub.execute_input":"2021-06-15T20:29:00.530350Z","iopub.status.idle":"2021-06-15T20:29:00.862625Z","shell.execute_reply.started":"2021-06-15T20:29:00.530319Z","shell.execute_reply":"2021-06-15T20:29:00.861635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Wow! What is target1, target2, target3, target4 mean?\n\n[Tyler Wayne Skaggs](https://en.wikipedia.org/wiki/Tyler_Skaggs) (July 13, 1991 – July 1, 2019) was an American left-handed professional baseball starting pitcher who played in Major League Baseball (MLB) for the Arizona Diamondbacks and Los Angeles Angels of Anaheim.","metadata":{}},{"cell_type":"code","source":"onDeceased = np.zeros((4,))\nonUnDeceased = np.zeros((4,))\nonDeceasedCount = 0\nonUnDeceasedCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    deceased = playerDeceased[pid]\n    tundeceased = target[deceased == 0]\n    tdeceased = target[deceased > 0]\n    \n    onDeceasedCount += 1 if len(tdeceased) > 0 else 0\n    onUnDeceasedCount += 1 if len(tundeceased) > 0 else 0\n    for j in range(0, 4):\n        onDeceased[j] += np.mean(tdeceased[:, j]) if len(tdeceased) > 0 else 0\n        onUnDeceased[j] += np.mean(tundeceased[:, j]) if len(tundeceased) > 0 else 0\nonDeceased /= onDeceasedCount\nonUnDeceased /= onUnDeceasedCount\n\n_ex = pd.DataFrame({'isDeceased': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onDeceased[0], onUnDeceased[0], onDeceased[1], onUnDeceased[1], onDeceased[2], onUnDeceased[2], onDeceased[3], onUnDeceased[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isDeceased', barmode='group')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:00.863923Z","iopub.execute_input":"2021-06-15T20:29:00.864224Z","iopub.status.idle":"2021-06-15T20:29:01.152055Z","shell.execute_reply.started":"2021-06-15T20:29:00.864195Z","shell.execute_reply":"2021-06-15T20:29:01.151001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3.2.5\"></a>\n### 3.2.5 Injured Feature","metadata":{}},{"cell_type":"code","source":"playerInjured = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nlastDate = None\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    updatedPid = set()\n    for r in json.loads(roster):\n        if r['playerId'] in playerIL:\n            if r['status'] == 'Injured 7-Day':\n                playerInjured[r['playerId']][i] += 7\n                updatedPid.add(r['playerId'])\n            elif r['status'] == 'Injured 10-Day':\n                playerInjured[r['playerId']][i] += 10\n                updatedPid.add(r['playerId'])\n            elif r['status'] == 'Injured 60-Day':\n                playerInjured[r['playerId']][i] += 60\n                updatedPid.add(r['playerId'])\n                \n    if lastDate is not None:\n        day = (train_df.date[i] - lastDate).days\n        for pid in playerInjured:\n            if pid not in updatedPid:\n                playerInjured[pid][i] += max(0, playerInjured[pid][i-1] - 1)\n    lastDate = train_df.date[i]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:01.153297Z","iopub.execute_input":"2021-06-15T20:29:01.153586Z","iopub.status.idle":"2021-06-15T20:29:14.020015Z","shell.execute_reply.started":"2021-06-15T20:29:01.153557Z","shell.execute_reply":"2021-06-15T20:29:14.018801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 640449\n_ex = pd.DataFrame({'date': train_df.date, 'Injured Days': playerInjured[pid], \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['Injured Days', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['Injured Days', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:14.021770Z","iopub.execute_input":"2021-06-15T20:29:14.022104Z","iopub.status.idle":"2021-06-15T20:29:14.356521Z","shell.execute_reply.started":"2021-06-15T20:29:14.022075Z","shell.execute_reply":"2021-06-15T20:29:14.355258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"injured = np.concatenate([v for v in playerInjured.values()], axis=0)\ntargets = np.concatenate([v[:, 0] for v in playerTarget.values()], axis=0)\nbins = np.array(list(range(0, 100)))\nvalue = np.array([np.sum(injured[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(injured)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target1', 'y':'injured'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:14.358285Z","iopub.execute_input":"2021-06-15T20:29:14.358723Z","iopub.status.idle":"2021-06-15T20:29:14.946674Z","shell.execute_reply.started":"2021-06-15T20:29:14.358677Z","shell.execute_reply":"2021-06-15T20:29:14.945704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = np.concatenate([v[:, 1] for v in playerTarget.values()], axis=0)\nvalue = np.array([np.sum(injured[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(injured)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target2', 'y':'injured'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:14.947838Z","iopub.execute_input":"2021-06-15T20:29:14.948107Z","iopub.status.idle":"2021-06-15T20:29:15.537715Z","shell.execute_reply.started":"2021-06-15T20:29:14.948079Z","shell.execute_reply":"2021-06-15T20:29:15.536591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = np.concatenate([v[:, 2] for v in playerTarget.values()], axis=0)\nvalue = np.array([np.sum(injured[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(injured)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target3', 'y':'injured'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:15.540569Z","iopub.execute_input":"2021-06-15T20:29:15.540951Z","iopub.status.idle":"2021-06-15T20:29:16.110796Z","shell.execute_reply.started":"2021-06-15T20:29:15.540920Z","shell.execute_reply":"2021-06-15T20:29:16.109771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = np.concatenate([v[:, 3] for v in playerTarget.values()], axis=0)\nvalue = np.array([np.sum(injured[(targets < b + 1) & (targets >= b)]) for b in bins])/np.sum(injured)\nfig = px.bar(x=bins, y=value, color=value, labels={'x':'target4', 'y':'injured'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:16.111974Z","iopub.execute_input":"2021-06-15T20:29:16.112322Z","iopub.status.idle":"2021-06-15T20:29:16.680767Z","shell.execute_reply.started":"2021-06-15T20:29:16.112292Z","shell.execute_reply":"2021-06-15T20:29:16.680037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Statistics for an injured feature is the same as for an illness feature.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2.5\"></a>\n### 3.2.5 Family Medical Emergency Feature","metadata":{}},{"cell_type":"code","source":"playerFME = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    for r in json.loads(roster):\n        if r['playerId'] in playerFME:\n            playerFME[r['playerId']][i] += int(r['status'] == 'Family Medical Emergency')\n            if r['status'] == 'Family Medical Emergency':\n                print('Family Medical Emergency Player:', r['playerId'], players_df.loc[players_df.playerId == r['playerId'], 'playerName'].values[0])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:16.682040Z","iopub.execute_input":"2021-06-15T20:29:16.682325Z","iopub.status.idle":"2021-06-15T20:29:22.479049Z","shell.execute_reply.started":"2021-06-15T20:29:16.682295Z","shell.execute_reply":"2021-06-15T20:29:22.477543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 456078\n_ex = pd.DataFrame({'date': train_df.date, 'isFME': playerFME[pid] * 10, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isFME', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isFME', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:22.480722Z","iopub.execute_input":"2021-06-15T20:29:22.481107Z","iopub.status.idle":"2021-06-15T20:29:22.824405Z","shell.execute_reply.started":"2021-06-15T20:29:22.481067Z","shell.execute_reply":"2021-06-15T20:29:22.823256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onFME = np.zeros((4,))\nonUnFME = np.zeros((4,))\nonFMECount = 0\nonUnFMECount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    fme = playerFME[pid]\n    tunfme = target[fme == 0]\n    tfme = target[fme > 0]\n    \n    onFMECount += 1 if len(tfme) > 0 else 0\n    onUnFMECount += 1 if len(tunfme) > 0 else 0\n    for j in range(0, 4):\n        onFME[j] += np.mean(tfme[:, j]) if len(tfme) > 0 else 0\n        onUnFME[j] += np.mean(tunfme[:, j]) if len(tunfme) > 0 else 0\nonFME /= onFMECount\nonUnFME /= onUnFMECount\n\n_ex = pd.DataFrame({'isFME': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onFME[0], onUnFME[0], onFME[1], onUnFME[1], onFME[2], onUnFME[2], onFME[3], onUnFME[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isFME', barmode='group')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:22.825908Z","iopub.execute_input":"2021-06-15T20:29:22.826208Z","iopub.status.idle":"2021-06-15T20:29:23.114230Z","shell.execute_reply.started":"2021-06-15T20:29:22.826180Z","shell.execute_reply":"2021-06-15T20:29:23.112997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hm... The average targets value is higher for players time periods with Family Medical Emergency Flag.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2.6\"></a>\n### 3.2.6 Paternity & Paternity List Feature","metadata":{}},{"cell_type":"code","source":"playerPaternity = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    for r in json.loads(roster):\n        if r['playerId'] in playerPaternity:\n            playerPaternity[r['playerId']][i] += int(r['status'] == 'Paternity' or r['status'] == 'Paternity List')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:23.115560Z","iopub.execute_input":"2021-06-15T20:29:23.115875Z","iopub.status.idle":"2021-06-15T20:29:28.649119Z","shell.execute_reply.started":"2021-06-15T20:29:23.115846Z","shell.execute_reply":"2021-06-15T20:29:28.647877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 628317\n_ex = pd.DataFrame({'date': train_df.date, 'isPaternity': playerPaternity[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isPaternity', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isPaternity', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:28.650324Z","iopub.execute_input":"2021-06-15T20:29:28.650625Z","iopub.status.idle":"2021-06-15T20:29:28.987569Z","shell.execute_reply.started":"2021-06-15T20:29:28.650597Z","shell.execute_reply":"2021-06-15T20:29:28.986361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onPaternity = np.zeros((4,))\nonUnPaternity = np.zeros((4,))\nonPaternityCount = 0\nonUnPaternityCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    paternity = playerPaternity[pid]\n    tunpaternity = target[paternity == 0]\n    tpaternity = target[paternity > 0]\n    \n    onPaternityCount += 1 if len(tpaternity) > 0 else 0\n    onUnPaternityCount += 1 if len(tunpaternity) > 0 else 0\n    for j in range(0, 4):\n        onPaternity[j] += np.mean(tpaternity[:, j]) if len(tpaternity) > 0 else 0\n        onUnPaternity[j] += np.mean(tunpaternity[:, j]) if len(tunpaternity) > 0 else 0\nonPaternity /= onPaternityCount\nonUnPaternity /= onUnPaternityCount\n\n_ex = pd.DataFrame({'isPaternity': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onPaternity[0], onUnPaternity[0], onPaternity[1], onUnPaternity[1], onPaternity[2], onUnPaternity[2], onPaternity[3], onUnPaternity[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isPaternity', barmode='group')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-15T20:29:28.989231Z","iopub.execute_input":"2021-06-15T20:29:28.989668Z","iopub.status.idle":"2021-06-15T20:29:29.288824Z","shell.execute_reply.started":"2021-06-15T20:29:28.989609Z","shell.execute_reply":"2021-06-15T20:29:29.287821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The average targets value is higher for players time periods with Paternity & Paternity List Flags.","metadata":{}},{"cell_type":"markdown","source":"<a id=\"3.2.7\"></a>\n### 3.2.7 Reassigned to Major Features","metadata":{}},{"cell_type":"code","source":"playerReassigned = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    roster = json.loads(roster)\n    for r in roster:\n        if r['playerId'] in playerReassigned:\n            playerReassigned[r['playerId']][i] += int(r['status'] == 'Reassigned' and r['status'] != 'Reassigned to Minors')\n            playerReassigned[r['playerId']][i] -= int(r['status'] == 'Reassigned to Minors')\n    for r in roster:\n        if r['playerId'] in playerReassigned:\n            playerReassigned[r['playerId']][i] = max(0., playerReassigned[r['playerId']][i])","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:29.290540Z","iopub.execute_input":"2021-06-15T20:29:29.290970Z","iopub.status.idle":"2021-06-15T20:29:42.507919Z","shell.execute_reply.started":"2021-06-15T20:29:29.290927Z","shell.execute_reply":"2021-06-15T20:29:42.506524Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 650619\n_ex = pd.DataFrame({'date': train_df.date, 'isReassignedToMajor': playerReassigned[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isReassignedToMajor', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isReassignedToMajor', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:42.509236Z","iopub.execute_input":"2021-06-15T20:29:42.509528Z","iopub.status.idle":"2021-06-15T20:29:42.857931Z","shell.execute_reply.started":"2021-06-15T20:29:42.509502Z","shell.execute_reply":"2021-06-15T20:29:42.856547Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onReassigned = np.zeros((4,))\nonUnReassigned = np.zeros((4,))\nonReassignedCount = 0\nonUnReassignedCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    reassigned = playerReassigned[pid]\n    tunreassigned = target[reassigned == 0]\n    treassigned = target[reassigned > 0]\n    \n    onReassignedCount += 1 if len(treassigned) > 0 else 0\n    onUnReassignedCount += 1 if len(tunreassigned) > 0 else 0\n    for j in range(0, 4):\n        onReassigned[j] += np.mean(treassigned[:, j]) if len(treassigned) > 0 else 0\n        onUnReassigned[j] += np.mean(tunreassigned[:, j]) if len(tunreassigned) > 0 else 0\nonReassigned /= onPaternityCount\nonUnReassigned /= onUnPaternityCount\n\n_ex = pd.DataFrame({'isReassignedToMajor': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onReassigned[0], onUnReassigned[0], onReassigned[1], onUnReassigned[1], onReassigned[2], onUnReassigned[2], onReassigned[3], onUnReassigned[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isReassignedToMajor', barmode='group')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:42.860401Z","iopub.execute_input":"2021-06-15T20:29:42.860921Z","iopub.status.idle":"2021-06-15T20:29:43.177035Z","shell.execute_reply.started":"2021-06-15T20:29:42.860860Z","shell.execute_reply":"2021-06-15T20:29:43.176022Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3.2.8\"></a>\n### 3.2.8 Reassigned  to Minor Features","metadata":{"execution":{"iopub.status.busy":"2021-06-15T19:44:03.1296Z","iopub.execute_input":"2021-06-15T19:44:03.13018Z","iopub.status.idle":"2021-06-15T19:44:03.137073Z","shell.execute_reply.started":"2021-06-15T19:44:03.130133Z","shell.execute_reply":"2021-06-15T19:44:03.135813Z"}}},{"cell_type":"code","source":"playerReassignedToMinor = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    roster = json.loads(roster)\n    for r in roster:\n        if r['playerId'] in playerReassignedToMinor:\n            playerReassignedToMinor[r['playerId']][i] += int(r['status'] == 'Reassigned to Minors')","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:43.184010Z","iopub.execute_input":"2021-06-15T20:29:43.184534Z","iopub.status.idle":"2021-06-15T20:29:49.020180Z","shell.execute_reply.started":"2021-06-15T20:29:43.184486Z","shell.execute_reply":"2021-06-15T20:29:49.018952Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 650619\n_ex = pd.DataFrame({'date': train_df.date, 'isReassignedToMinor': playerReassignedToMinor[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isReassignedToMinor', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isReassignedToMinor', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:49.021743Z","iopub.execute_input":"2021-06-15T20:29:49.022071Z","iopub.status.idle":"2021-06-15T20:29:49.361505Z","shell.execute_reply.started":"2021-06-15T20:29:49.022007Z","shell.execute_reply":"2021-06-15T20:29:49.360438Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onReassigned = np.zeros((4,))\nonUnReassigned = np.zeros((4,))\nonReassignedCount = 0\nonUnReassignedCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    reassigned = playerReassignedToMinor[pid]\n    tunreassigned = target[reassigned == 0]\n    treassigned = target[reassigned > 0]\n    \n    onReassignedCount += 1 if len(treassigned) > 0 else 0\n    onUnReassignedCount += 1 if len(tunreassigned) > 0 else 0\n    for j in range(0, 4):\n        onReassigned[j] += np.mean(treassigned[:, j]) if len(treassigned) > 0 else 0\n        onUnReassigned[j] += np.mean(tunreassigned[:, j]) if len(tunreassigned) > 0 else 0\nonReassigned /= onPaternityCount\nonUnReassigned /= onUnPaternityCount\n\n_ex = pd.DataFrame({'isReassignedToMinor': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onReassigned[0], onUnReassigned[0], onReassigned[1], onUnReassigned[1], onReassigned[2], onUnReassigned[2], onReassigned[3], onUnReassigned[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isReassignedToMinor', barmode='group')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:49.362739Z","iopub.execute_input":"2021-06-15T20:29:49.363020Z","iopub.status.idle":"2021-06-15T20:29:49.733533Z","shell.execute_reply.started":"2021-06-15T20:29:49.362993Z","shell.execute_reply":"2021-06-15T20:29:49.732555Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3.2.9\"></a>\n### 3.2.9 Reserve List (Minors) Features","metadata":{}},{"cell_type":"code","source":"playerReserve = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    roster = json.loads(roster)\n    for r in roster:\n        if r['playerId'] in playerReserve:\n            playerReserve[r['playerId']][i] += int(r['status'] == 'Reserve List (Minors)')","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:49.735333Z","iopub.execute_input":"2021-06-15T20:29:49.735614Z","iopub.status.idle":"2021-06-15T20:29:55.383073Z","shell.execute_reply.started":"2021-06-15T20:29:49.735587Z","shell.execute_reply":"2021-06-15T20:29:55.382380Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 656887\n_ex = pd.DataFrame({'date': train_df.date, 'isReserve': playerReserve[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isReserve', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isReserve', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:29:55.384182Z","iopub.execute_input":"2021-06-15T20:29:55.384612Z","iopub.status.idle":"2021-06-15T20:29:55.740311Z","shell.execute_reply.started":"2021-06-15T20:29:55.384563Z","shell.execute_reply":"2021-06-15T20:29:55.739235Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onReserve = np.zeros((4,))\nonUnReserve = np.zeros((4,))\nonReserveCount = 0\nonUnReserveCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    reserve = playerReserve[pid]\n    tunreserve = target[reserve == 0]\n    treserve = target[reserve > 0]\n    \n    onReserveCount += 1 if len(treserve) > 0 else 0\n    onUnReserveCount += 1 if len(tunreserve) > 0 else 0\n    for j in range(0, 4):\n        onReserve[j] += np.mean(treserve[:, j]) if len(treserve) > 0 else 0\n        onUnReserve[j] += np.mean(tunreserve[:, j]) if len(tunreserve) > 0 else 0\nonReserve /= onReserveCount\nonUnReserve /= onUnReserveCount\n\n_ex = pd.DataFrame({'isReserve': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onReserve[0], onUnReserve[0], onReserve[1], onUnReserve[1], onReserve[2], onUnReserve[2], onReserve[3], onUnReserve[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isReserve', barmode='group')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:32:29.242036Z","iopub.execute_input":"2021-06-15T20:32:29.242822Z","iopub.status.idle":"2021-06-15T20:32:29.585617Z","shell.execute_reply.started":"2021-06-15T20:32:29.242768Z","shell.execute_reply":"2021-06-15T20:32:29.584376Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3.2.10\"></a>\n### 3.2.10 Suspended Features","metadata":{}},{"cell_type":"code","source":"playerSuspended = {pid: np.zeros((len(train_df),)) for pid in players_df.playerId}\nfor i, roster in tqdm(enumerate(train_df.rosters)):\n    roster = json.loads(roster)\n    for r in roster:\n        if r['playerId'] in playerSuspended:\n            playerSuspended[r['playerId']][i] += int(r['status'] == 'Suspended' or r['status'] == 'Suspended # days')","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:41:03.414740Z","iopub.execute_input":"2021-06-15T20:41:03.415075Z","iopub.status.idle":"2021-06-15T20:41:09.583929Z","shell.execute_reply.started":"2021-06-15T20:41:03.415045Z","shell.execute_reply":"2021-06-15T20:41:09.583184Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid = 592206\n_ex = pd.DataFrame({'date': train_df.date, 'isSuspended': playerSuspended[pid] * 100, \n                    'Target1': playerTarget[pid][:, 0], 'Target2': playerTarget[pid][:, 1],\n                    'Target3': playerTarget[pid][:, 2], 'Target4': playerTarget[pid][:, 3]})\nfig = px.line(_ex, x='date', y=['isSuspended', 'Target1', 'Target2', 'Target3', 'Target4'])\nfig.for_each_trace(lambda trace: trace.update(visible=\"legendonly\") \n                   if trace.name not in ['isSuspended', 'Target1'] else ())\nfig.update_layout(\n    title={\n        'text': f\"PlayerId: {pid}; Player Name: {players_df.loc[players_df.playerId == pid, 'playerName'].values[0]}\",\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:41:09.585152Z","iopub.execute_input":"2021-06-15T20:41:09.585895Z","iopub.status.idle":"2021-06-15T20:41:09.929737Z","shell.execute_reply.started":"2021-06-15T20:41:09.585848Z","shell.execute_reply":"2021-06-15T20:41:09.928476Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onSuspended = np.zeros((4,))\nonUnSuspended = np.zeros((4,))\nonSuspendedCount = 0\nonUnSuspendedCount = 0\nfor pid in playerTarget:\n    target = playerTarget[pid]\n    suspended = playerSuspended[pid]\n    tunsuspended = target[suspended == 0]\n    tsuspended = target[suspended > 0]\n    \n    onSuspendedCount += 1 if len(tsuspended) > 0 else 0\n    onUnSuspendedCount += 1 if len(tunsuspended) > 0 else 0\n    for j in range(0, 4):\n        onSuspended[j] += np.mean(tsuspended[:, j]) if len(tsuspended) > 0 else 0\n        onUnSuspended[j] += np.mean(tsuspended[:, j]) if len(tsuspended) > 0 else 0\nonSuspended /= onSuspendedCount\nonUnSuspended /= onUnSuspendedCount\n\n_ex = pd.DataFrame({'isReserve': [True, False, True, False, True, False, True, False], \n                    'targetMean': [onSuspended[0], onUnSuspended[0], onSuspended[1], onUnSuspended[1], onSuspended[2], onUnSuspended[2], onSuspended[3], onUnSuspended[3]],\n                    'targetType': ['target1', 'target1', 'target2', 'target2', 'target3', 'target3', 'target4', 'target4']\n                   })\nfig = px.bar(_ex, x='targetType', y='targetMean', color='isReserve', barmode='group')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T20:45:07.646332Z","iopub.execute_input":"2021-06-15T20:45:07.646852Z","iopub.status.idle":"2021-06-15T20:45:07.855746Z","shell.execute_reply.started":"2021-06-15T20:45:07.646809Z","shell.execute_reply":"2021-06-15T20:45:07.854674Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### to be continued...","metadata":{}}]}