{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport gc\nfrom tqdm import tqdm\n\nimport pickle\nimport numpy as np\nfrom functools import reduce\nfrom datetime import datetime, timedelta\nfrom typing import Callable, Dict, List, Tuple\n\ndef load_obj(name):\n    with open(\"../input/baseball/\" + name + \".pkl\", \"rb\") as f:\n        return pickle.load(f)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-23T15:07:31.465538Z","iopub.execute_input":"2021-07-23T15:07:31.465992Z","iopub.status.idle":"2021-07-23T15:07:31.479318Z","shell.execute_reply.started":"2021-07-23T15:07:31.465899Z","shell.execute_reply":"2021-07-23T15:07:31.477936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_factorize(vitrine: pd.DataFrame, field: str) -> pd.DataFrame:\n    code, cats = pd.factorize(vitrine[field])\n    vitrine[field] = code\n    return vitrine\n\n\ndef decrease_mem_consuming(\n    features: pd.DataFrame, excluding_fields: List[str] = [\"id\", \"date\"]\n) -> pd.DataFrame:\n\n    new_types = {}\n    for name in list(features.columns):\n        if name in excluding_fields:\n            continue\n        if features[name].dtype == \"float64\":\n            new_types[name] = \"float32\"\n        elif features[name].dtype == \"int64\":\n            new_types[name] = \"int32\"\n\n    features = features.astype(new_types)\n    return features\n\n\n\n\nd = load_obj('prod4_1f')\nmodels = load_obj('full_target_1_354')\nnames_nec_fields = load_obj('names_nec_fields')\n\n\n\npls = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/players.csv')\n\nsns = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/seasons.csv')\nsns = sns.rename(columns={\"seasonId\": \"season\"})\nsns = decrease_mem_consuming(sns)\n\n\n\ntms = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/teams.csv')\ntms = tms.drop(\n    [\n        \"name\",\n        \"teamName\",\n        \"teamCode\",\n        \"shortName\",\n        \"abbreviation\",\n        \"venueId\",\n        \"venueName\",\n        \"leagueName\",\n        \"divisionName\",\n        \"divisionId\"\n    ],\n    axis=1,\n)\ntms = tms.rename(\n    columns={\n        \"id\": \"team_id\",\n    }\n)\ntms = get_factorize(tms, \"locationName\")\ntms = decrease_mem_consuming(tms)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T15:07:31.481278Z","iopub.execute_input":"2021-07-23T15:07:31.482034Z","iopub.status.idle":"2021-07-23T15:07:32.652050Z","shell.execute_reply.started":"2021-07-23T15:07:31.481987Z","shell.execute_reply":"2021-07-23T15:07:32.650919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extend_specific_field(df: pd.DataFrame, field_name: str) -> pd.DataFrame:\n    df = df.query(f\"{field_name} == {field_name}\")\n    null = np.nan\n    true = True\n    false = False\n    res = []\n    for el in list(df[field_name]):\n        res += eval(el)\n    return res\n\n\ndef get_df_from_extend_field(\n    train: pd.DataFrame, field_name: str, names: List[str], drop_list: List[str]\n) -> pd.DataFrame:\n    df = extend_specific_field(train, field_name)\n    df = pd.DataFrame(df)\n    if len(drop_list) != 0:\n        df = df.drop(drop_list, axis=1)\n    if names != []:\n        df.columns = names\n    return df\n\n\ndef get_object_types(df: pd.DataFrame) -> List[str]:\n    return list(\n        set(filter(lambda name: df[name].dtype == object, df.columns)) - set([\"date\"])\n    )\n\n\n\ndef get_part_of_vitrine(\n    df: pd.DataFrame,\n    vitrine: pd.DataFrame,\n    merge_fields: List[str],\n    nec_fields: List[str],\n) -> pd.DataFrame:\n    df = decrease_mem_consuming(df)\n    types = df.dtypes.to_dict()\n    vitrine = pd.merge(vitrine, df, on=merge_fields, how=\"left\")\n    vitrine = vitrine.replace([np.inf, -np.inf], np.nan)\n    vitrine = vitrine.fillna(-1)\n    vitrine = vitrine.groupby([\"id\", \"date\"]).sum().reset_index()\n    vitrine = vitrine.astype(types)\n        \n    return vitrine[nec_fields]\n\n\ndef generate_rosters(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    fields = names_nec_fields['rosters']\n    \n    if train['rosters'].iloc[0] == train['rosters'].iloc[0]:\n        rosters = get_df_from_extend_field(\n            train, \"rosters\", [\"id\", \"date\"] + [\"team_id\", \"status\"], [\"status\"]\n        )\n        rosters = get_factorize(rosters, \"status\")\n        rosters.status = rosters.status.replace(\n            {\n                4: 3,  # Injured 7-Day -> Injured 10-Day\n                5: 0,\n                6: 0,\n                7: 0,\n                8: 0,\n                9: 0,\n                10: 0,\n            }\n        )\n    else:\n        rosters = vitrine.copy()\n        for col in fields:\n            rosters[col] = np.nan\n    rosters = rosters[['id', 'date'] + fields]\n    vitrine = get_part_of_vitrine(\n        rosters, vitrine, [\"id\", \"date\"], [\"id\", \"date\", \"team_id\", \"status\"]\n    )\n    return vitrine.fillna(-1)\n\ndef prepare_games(df: pd.DataFrame) -> pd.DataFrame:\n    df[\"homeWinner\"] = df[\"homeWinner\"].astype(int)\n    df[\"awayWinner\"] = df[\"awayWinner\"].astype(int)\n    df[\"isTie\"] = df[\"isTie\"].fillna(False)\n    df[\"isTie\"] = df[\"isTie\"].astype(int)\n\n    # Generate rel features for game\n    df[\"rel_score\"] = df[\"homeScore\"] / df[\"awayScore\"]\n    df[\"rel_win_pct\"] = df[\"homeWinPct\"] / df[\"awayWinPct\"]\n    df[\"rel_wins\"] = df[\"homeWins\"] / df[\"awayWins\"]\n    df[\"rel_losses\"] = df[\"homeLosses\"] / df[\"awayLosses\"]\n\n    df[\"home_team\"] = df[\"homeId\"]\n    df[\"away_team\"] = df[\"awayId\"]\n\n    df_1 = df.drop(\n        [\"awayId\", \"awayWins\", \"awayLosses\", \"awayWinPct\", \"awayWinner\", \"awayScore\"],\n        axis=1,\n    )\n    df_1 = df_1.rename(\n        columns={\n            \"homeId\": \"team_id\",\n            \"homeWins\": \"count_wins_on_season\",\n            \"homeLosses\": \"count_losses_on_season\",\n            \"homeWinPct\": \"win_pct\",\n            \"homeWinner\": \"win\",\n            \"homeScore\": \"score\",\n            \"gameDate\": \"date\",\n        }\n    )\n    df_1.loc[:, \"is_home\"] = 1\n\n    df_2 = df.drop(\n        [\"homeId\", \"homeWins\", \"homeLosses\", \"homeWinPct\", \"homeWinner\", \"homeScore\"],\n        axis=1,\n    )\n    df_2 = df_2.rename(\n        columns={\n            \"awayId\": \"team_id\",\n            \"awayWins\": \"count_wins_on_season\",\n            \"awayLosses\": \"count_losses_on_season\",\n            \"awayWinPct\": \"win_pct\",\n            \"awayWinner\": \"win\",\n            \"awayScore\": \"score\",\n            \"gameDate\": \"date\",\n        }\n    )\n    df_2.loc[:, \"is_home\"] = 0\n\n    # Generate rel features for game\n    df_2[\"rel_score\"] = 1 / df_2[\"rel_score\"]\n    df_2[\"rel_win_pct\"] = 1 / df_2[\"rel_win_pct\"]\n    df_2[\"rel_wins\"] = 1 / df_2[\"rel_wins\"]\n    df_2[\"rel_losses\"] = 1 / df_2[\"rel_losses\"]\n\n    res = pd.concat([df_1, df_2]).reset_index(drop=True)\n\n    index_resumed = res.query(\"resumeDate == resumeDate\").index\n\n    df_3 = res.loc[index_resumed, :]\n    df_3 = df_3.drop([\"date\"], axis=1)\n    df_3 = df_3.rename(columns={\"resumeDate\": \"date\"})\n    df_3[\"date\"] = df_3[\"date\"].apply(lambda x: x[:10])\n    df_3[\"is_resume\"] = 2\n\n    for name in [\n        \"isTie\",\n        \"count_wins_on_season\",\n        \"count_losses_on_season\",\n        \"win_pct\",\n        \"win\",\n        \"score\",\n        \"rel_score\",\n        \"rel_win_pct\",\n        \"rel_wins\",\n        \"rel_losses\",\n    ]:\n        res.loc[index_resumed, name] = -1\n\n    res.loc[index_resumed, \"is_resume\"] = 1\n    res[\"is_resume\"] = res[\"is_resume\"].fillna(-1)\n\n    res = pd.concat([res, df_3]).reset_index(drop=True)\n\n    res.loc[:, \"is_game\"] = 1\n\n    res = res.rename(columns={\"gamePk\": \"game_id\"})\n\n    res = res[\n        [\n            \"team_id\",\n            \"game_id\",\n            \"home_team\",\n            \"away_team\",\n            \"date\",\n            \"is_game\",\n            \"is_resume\",\n            \"win\",\n            \"score\",\n            \"is_home\",\n            \"win_pct\",\n            \"count_wins_on_season\",\n            \"count_losses_on_season\",\n            \"gameType\",\n            \"codedGameState\",\n            \"detailedGameState\",\n            \"isTie\",\n            \"gameNumber\",\n            \"doubleHeader\",\n            \"gamesInSeries\",\n            \"rel_score\",\n            \"rel_win_pct\",\n            \"rel_wins\",\n            \"rel_losses\",\n        ]\n    ]\n    return res\n\ndef prepare_games_field(vitrine: pd.DataFrame) -> pd.DataFrame:\n    vitrine.gameType = vitrine.gameType.replace({\"F\": \"D\", \"W\": \"L\"})\n    vitrine = get_factorize(vitrine, \"gameType\")\n    vitrine = get_factorize(vitrine, \"codedGameState\")\n    vitrine = get_factorize(vitrine, \"detailedGameState\")\n    vitrine = get_factorize(vitrine, \"doubleHeader\")\n\n    vitrine.is_game = vitrine.is_game.fillna(-1)\n    vitrine.gameType = vitrine.gameType.fillna(-1)\n    vitrine.codedGameState = vitrine.codedGameState.fillna(-1)\n    vitrine.detailedGameState = vitrine.detailedGameState.fillna(-1)\n    vitrine.isTie = vitrine.isTie.fillna(-1)\n    vitrine.doubleHeader = vitrine.doubleHeader.fillna(-1)\n\n    for name in [\n        \"is_resume\",\n        \"win\",\n        \"score\",\n        \"is_home\",\n        \"win_pct\",\n        \"count_wins_on_season\",\n        \"count_losses_on_season\",\n        \"gameNumber\",\n        \"gamesInSeries\",\n        \"rel_score\",\n        \"rel_win_pct\",\n        \"rel_wins\",\n        \"rel_losses\",\n        \"home_team\",\n        \"away_team\",\n    ]:\n        vitrine[name] = vitrine[name].fillna(-1)\n    for name in [\n        \"is_resume\",\n        \"count_wins_on_season\",\n        \"count_losses_on_season\",\n        \"gamesInSeries\",\n    ]:\n        vitrine[name] = vitrine[name].astype(int)\n    return vitrine\n\n\ndef generate_games(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    fields = names_nec_fields['games']\n    nec_fields = list(set(fields) - set([\"home_team\", \"away_team\"])) + [\"enemy_team\"]\n    if train['games'].iloc[0] == train['games'].iloc[0]:\n        games = get_df_from_extend_field(\n            train,\n            \"games\",\n            [],\n            [],\n        )\n        games = prepare_games(games)\n        games = prepare_games_field(games)\n\n        features = get_part_of_vitrine(\n            games,\n            vitrine,\n            [\"team_id\", \"date\"],\n            ['id', 'date', 'game_id', 'team_id'] + fields\n        )\n\n        features.loc[:, \"enemy_team\"] = -1\n        is_home_index = features.query(\"is_home == 1\").index\n        not_is_home_index = features.query(\"is_home == 0\").index\n        features.loc[is_home_index, \"enemy_team\"] = features.loc[is_home_index, \"away_team\"]\n        features.loc[not_is_home_index, \"enemy_team\"] = features.loc[\n            not_is_home_index, \"home_team\"\n        ]\n        features = features.drop([\"home_team\", \"away_team\"], axis=1)\n    else:\n        features = vitrine.copy()\n        for col in nec_fields:\n            features[col] = np.nan\n    features = features[['id', 'date', 'game_id', 'team_id'] + nec_fields]\n        \n    return features.fillna(-1)\n\n\ndef prepare_player_box_score(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.rename(columns={\"gameDate\": \"date\", \"playerId\": \"id\", \"gamePk\": \"game_id\"})\n    names = list(set(df.columns) - set([\"id\", \"date\", \"game_id\"]))\n    df.positionCode = df.positionCode.apply(eval)\n    df.jerseyNum = df.jerseyNum.apply(eval_str)\n    df.jerseyNum = df.jerseyNum.replace(\n        [\n            69.0,\n            72.0,\n            73.0,\n            75.0,\n            76.0,\n            78.0,\n            79.0,\n            80.0,\n            81.0,\n            82.0,\n            83.0,\n            84.0,\n            85.0,\n            86.0,\n            87.0,\n            88.0,\n            89.0,\n            90.0,\n            91.0,\n            92.0,\n            93.0,\n            94.0,\n            95.0,\n            96.0,\n        ],\n        -1,\n    )\n    df = get_factorize(df, \"positionType\")\n    for name in [\n        \"flyOutsPitching\",\n        \"gamesPlayedBatting\",\n        \"sacFliesPitching\",\n        \"blownSaves\",\n        \"saveOpportunities\",\n        \"assists\",\n        \"putOuts\",\n        \"sacBuntsPitching\",\n        \"hits\",\n        \"groundOutsPitching\",\n        \"doubles\",\n        \"leftOnBase\",\n        \"gamesFinishedPitching\",\n        \"balls\",\n        \"strikes\",\n        \"pickoffs\",\n        \"hitByPitch\",\n        \"lossesPitching\",\n        \"gamesStartedPitching\",\n        \"inheritedRunnersScored\",\n        \"wildPitches\",\n        \"atBatsPitching\",\n        \"sacBunts\",\n        \"strikeOutsPitching\",\n        \"catchersInterference\",\n        \"hitsPitching\",\n        \"catchersInterferencePitching\",\n        \"runsScored\",\n        \"baseOnBalls\",\n        \"gamesPlayedPitching\",\n        \"errors\",\n        \"rbi\",\n        \"rbiPitching\",\n        \"balks\",\n        \"caughtStealing\",\n        \"shutoutsPitching\",\n        \"groundIntoTriplePlay\",\n        \"plateAppearances\",\n        \"hitBatsmen\",\n        \"inningsPitched\",\n        \"pickoffsPitching\",\n        \"pitchesThrown\",\n        \"groundIntoDoublePlay\",\n        \"flyOuts\",\n        \"homeRunsPitching\",\n        \"homeRuns\",\n        \"chances\",\n        \"stolenBases\",\n        \"airOutsPitching\",\n        \"outsPitching\",\n        \"caughtStealingPitching\",\n        \"holds\",\n        \"strikeOuts\",\n        \"hitByPitchPitching\",\n        \"runsPitching\",\n        \"intentionalWalks\",\n        \"jerseyNum\",\n        \"totalBases\",\n        \"stolenBasesPitching\",\n        \"saves\",\n        \"intentionalWalksPitching\",\n        \"inheritedRunners\",\n        \"battersFaced\",\n        \"groundOuts\",\n        \"triples\",\n        \"earnedRuns\",\n        \"battingOrder\",\n        \"baseOnBallsPitching\",\n        \"doublesPitching\",\n        \"sacFlies\",\n        \"triplesPitching\",\n        \"winsPitching\",\n        \"atBats\",\n        \"completeGamesPitching\",\n    ]:\n        df[name] = df[name].fillna(-1).astype(int)\n\n    return df.loc[:, [\"id\", \"date\", \"game_id\"] + names]\n\ndef eval_str(x):\n    if x == \"\":\n        return np.nan\n    if isinstance(x, str):\n        return int(eval(x))\n    return np.nan\n\ndef generate_player_box(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    nec_fields = names_nec_fields['player_box']\n    if train['playerBoxScores'].iloc[0] == train['playerBoxScores'].iloc[0]:\n        player_box = get_df_from_extend_field(\n            train,\n            \"playerBoxScores\",\n            [],\n            [\"gameTimeUTC\", \"teamName\", \"playerName\", \"positionName\", \"teamId\"],\n        )\n        player_box = prepare_player_box_score(player_box)\n        #print(player_box.columns)\n\n        features = get_part_of_vitrine(\n            player_box,\n            vitrine,\n            [\"id\", \"date\", \"game_id\"],\n            [\"id\", \"date\", 'game_id', 'team_id'] + nec_fields\n        )\n    else:\n        features = vitrine.copy()\n        for col in nec_fields:\n            features[col] = np.nan\n    features = features[['id', 'date', 'game_id', 'team_id'] + nec_fields]\n        \n    return features.fillna(-1)\n\ndef prepare_team_box_score(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.rename(columns={\"gamePk\": \"game_id\", \"teamId\": \"team_id\"})\n    names = list(set(df.columns) - set([\"team_id\", \"game_id\"]))\n    df = df.loc[:, [\"team_id\", \"game_id\"] + names]\n    names = list(map(lambda x: x + \"_team\", names))\n    df.columns = [\"team_id\", \"game_id\"] + names\n\n    return df\n\ndef generate_team_box(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    nec_fields = names_nec_fields['team_box']\n    if train['teamBoxScores'].iloc[0] == train['teamBoxScores'].iloc[0]:\n        team_box = get_df_from_extend_field(\n            train,\n            \"teamBoxScores\",\n            [],\n            [\"home\", \"gameTimeUTC\", \"gameDate\"],\n        )\n        team_box = prepare_team_box_score(team_box)\n\n        features = get_part_of_vitrine(\n            team_box,\n            vitrine,\n            [\"team_id\", \"game_id\"],\n            [\"date\", \"id\", \"team_id\", \"game_id\"] + nec_fields,\n        )\n    else:\n        features = vitrine.copy()\n        for col in nec_fields:\n            features[col] = np.nan\n    features = features[['id', 'date', \"team_id\", 'game_id'] + nec_fields]\n        \n    return features.fillna(-1)\n\n\ndef get_types_dict_rename(types: List[str], sub_date: str, sub_field: str) -> Dict:\n    res = dict()\n    for el in types:\n        res[el] = f\"{el}_{sub_date}_{sub_field}\"\n    return res\n\ndef add_transaction_features(\n    vitrine: pd.DataFrame,\n    df_i: pd.DataFrame,\n    old_name_date: str = \"date\",\n    sub_date: str = \"simple\",\n) -> pd.DataFrame:\n    df = df_i.rename(columns={\"date\": \"i_date\"}).rename(columns={old_name_date: \"date\"})\n\n    keys = [[\"date\", \"team_id\"], [\"date\", \"team_id\"], [\"date\", \"id\"]]\n    names = [\"from\", \"to\", \"player\"]\n    types = [\n        \"SFA\",\n        \"TR\",\n        \"NUM\",\n        \"ASG\",\n        \"DES\",\n        \"CLW\",\n        \"OUT\",\n        \"REL\",\n        \"SC\",\n        \"OPT\",\n        \"RTN\",\n        \"SGN\",\n        \"SE\",\n        \"CU\",\n        \"DFA\",\n        \"RET\",\n    ]\n    for t in types:\n        if not t in list(df.columns):\n            df[t] = 0\n    for ind, second_field in enumerate([\"fromTeamId\", \"toTeamId\", \"playerId\"]):\n        df_curr = df.groupby([\"date\", second_field]).sum().reset_index()\n        df_curr = df_curr.astype({second_field: \"int64\"})\n        df_curr = df_curr[[\"date\", second_field] + types]\n        df_curr = df_curr.rename(\n            columns=get_types_dict_rename(types, sub_date, names[ind])\n        )\n        vitrine = pd.merge(\n            vitrine,\n            df_curr,\n            left_on=keys[ind],\n            right_on=[\"date\", second_field],\n            how=\"left\",\n        )\n        vitrine = vitrine.drop([second_field], axis=1)\n\n    return vitrine\n\ndef generate_trans(train: pd.DataFrame, vitrine_i: pd.DataFrame) -> pd.DataFrame:\n    nec_fields = names_nec_fields['trans']\n    vitrine = vitrine_i.copy()\n    if train['transactions'].iloc[0] == train['transactions'].iloc[0]:\n        trans = get_df_from_extend_field(\n            train,\n            \"transactions\",\n            [],\n            [\n                \"transactionId\",\n                \"playerName\",\n                \"fromTeamName\",\n                \"toTeamName\",\n                \"description\",\n                \"typeDesc\",\n            ],\n        )\n        trans = pd.concat([trans, pd.get_dummies(trans[\"typeCode\"])], axis=1)\n        trans = trans.drop([\"typeCode\"], axis=1)\n\n        sub_dates = [\"simple\", \"eff\", \"resol\"]\n        for ind, date_type in enumerate([\"i_date\", \"effectiveDate\", \"resolutionDate\"]):\n            vitrine = add_transaction_features(vitrine, trans, date_type, sub_dates[ind])\n        for name in set(vitrine.columns) - set([\"team_id\", \"game_id\", \"id\", \"date\"]):\n            vitrine[name] = vitrine[name].fillna(-1).astype(int)\n\n        features = vitrine\n    else:\n        features = vitrine\n        for col in nec_fields:\n            features[col] = np.nan\n    features = features[['id', 'date', \"team_id\", 'game_id'] + nec_fields]\n        \n    return features.fillna(-1)\n\n\n\n\n\n\ndef preapre_standings(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.rename(columns={\"gameDate\": \"date\", \"teamId\": \"team_id\"})\n    df = get_factorize(df, \"streakCode\")\n    df[\"wildCardLeader\"] = df[\"wildCardLeader\"].fillna(\"False\")\n\n    for name in get_object_types(df):\n        df[name] = df[name].replace([\"-\", \"E\"], \"-1\").fillna(\"-1\").apply(eval)\n\n    for name in [\n        \"wildCardLeader\",\n        \"divisionLeader\",\n        \"divisionChamp\",\n        \"alWins\",\n        \"alLosses\",\n        \"nlWins\",\n        \"nlLosses\",\n    ]:\n        df[name] = df[name].fillna(-1).astype(int)\n\n    return df\n\n\n\ndef generate_standings(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    nec_fields = names_nec_fields['standings']\n    if train['standings'].iloc[0] == train['standings'].iloc[0]:\n        standings = get_df_from_extend_field(\n            train,\n            \"standings\",\n            [],\n            [\n                \"season\",\n                \"teamName\",\n            ],\n        )\n        standings = preapre_standings(standings)\n        features = get_part_of_vitrine(\n            standings,\n            vitrine,\n            [\"team_id\", \"date\"],\n            [\"date\", \"id\", \"team_id\", \"game_id\"] + nec_fields,\n        )\n    else:\n        features = vitrine.copy()\n        for col in nec_fields:\n            features[col] = np.nan\n    features = features[['id', 'date', \"team_id\", 'game_id'] + nec_fields]\n        \n    return features.fillna(-1)\n\n\ndef preapre_awards(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.rename(columns={\"awardDate\": \"date\", \"playerId\": \"id\"})\n    df = get_factorize(df, \"awardId\")\n    df.loc[:, \"is_award\"] = 1\n    return df\n\n\ndef generate_awards(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    nec_fields = names_nec_fields['awards']\n    if train['awards'].iloc[0] == train['awards'].iloc[0]:\n        awards = get_df_from_extend_field(\n            train,\n            \"awards\",\n            [],\n            [\"awardName\", \"awardSeason\", \"playerName\", \"awardPlayerTeamId\"],\n        )\n        awards = preapre_awards(awards)\n        features = get_part_of_vitrine(\n            awards,\n            vitrine,\n            [\"id\", \"date\"],\n            [\"date\", \"id\", \"team_id\", \"game_id\"] + nec_fields,\n        )\n    else:\n        features = vitrine.copy()\n        for col in nec_fields:\n            features[col] = np.nan\n    features = features[['id', 'date', \"team_id\", 'game_id'] + nec_fields]\n        \n    return features.fillna(-1)\n\n\ndef preapre_player_twit(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.rename(\n        columns={\"playerId\": \"id\", \"numberOfFollowers\": \"count_follow_player\"}\n    )\n    return df\n\n\ndef generate_player_twit(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    if train['playerTwitterFollowers'].iloc[0] == train['playerTwitterFollowers'].iloc[0]:\n        player_twit = get_df_from_extend_field(\n            train,\n            \"playerTwitterFollowers\",\n            [],\n            [\n                \"playerName\",\n                \"accountName\",\n                \"twitterHandle\",\n            ],\n        )\n        player_twit = preapre_player_twit(player_twit)\n        player_twit = player_twit[['id', \"count_follow_player\"]]\n        player_twit.index = player_twit['id']\n        player_twit = player_twit.to_dict('index')\n        return player_twit\n    else:\n        return dict()\n    \n    \ndef preapre_team_twit(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.rename(\n        columns={\"teamId\": \"team_id\", \"numberOfFollowers\": \"count_follow_team\"}\n    )\n    return df\n\n\ndef generate_team_twit(train: pd.DataFrame, vitrine: pd.DataFrame) -> pd.DataFrame:\n    if train['playerTwitterFollowers'].iloc[0] == train['playerTwitterFollowers'].iloc[0]:\n        team_twit = get_df_from_extend_field(\n            train,\n            \"teamTwitterFollowers\",\n            [],\n            [\n                \"teamName\",\n                \"accountName\",\n                \"twitterHandle\",\n            ],\n        )\n        team_twit = preapre_team_twit(team_twit)\n        team_twit = team_twit[['team_id', \"count_follow_team\"]]\n        team_twit.index = team_twit['team_id']\n        team_twit = team_twit.to_dict('index')\n        return team_twit\n    else:\n        return dict()\n    \n    \ndef add_time_features(df: pd.DataFrame) -> pd.DataFrame:\n    df[\"age\"] = (pd.to_datetime(df[\"date\"]) - pd.to_datetime(df[\"DOB\"])).dt.days / 365\n    df[\"year_after_debut\"] = (\n        pd.to_datetime(df[\"date\"]) - pd.to_datetime(df[\"mlbDebutDate\"])\n    ).dt.days / 365\n    df[\"debut_age\"] = (\n        pd.to_datetime(df[\"mlbDebutDate\"]) - pd.to_datetime(df[\"DOB\"])\n    ).dt.days / 365\n    df[\"rel_mlb_age\"] = df[\"year_after_debut\"] / df[\"age\"]\n\n    return df.drop([\"DOB\", \"mlbDebutDate\"], axis=1)\n\n\ndef generate_players(vitrine_i: pd.DataFrame) -> pd.DataFrame:\n    vitrine = vitrine_i.copy()\n    players = pls.copy()\n    players = players.drop(\n        [\n            \"playerName\",\n            \"birthCity\",\n            \"birthStateProvince\",\n            \"playerForTestSetAndFuturePreds\",\n        ],\n        axis=1,\n    )\n    players = players.rename(\n        columns={\n            \"playerId\": \"id\",\n        }\n    )\n    players = get_factorize(players, \"birthCountry\")\n    players = get_factorize(players, \"primaryPositionName\")\n    players[\"primaryPositionCode\"] = (\n        players[\"primaryPositionCode\"]\n        .replace({\"I\": \"11\", \"O\": \"0\"})\n        .fillna(\"-1\")\n        .apply(eval)\n    )\n\n    players = decrease_mem_consuming(players)\n    types = players.dtypes.to_dict()\n    del types[\"DOB\"]\n    del types[\"mlbDebutDate\"]\n\n    vitrine = pd.merge(\n        vitrine[[\"id\", \"date\", \"game_id\", \"team_id\"]], players, on=[\"id\"], how=\"left\"\n    )\n    vitrine = vitrine.replace([np.inf, -np.inf], np.nan)\n    vitrine = vitrine.fillna(-1)\n    vitrine = add_time_features(vitrine)\n    vitrine = vitrine.astype(types)\n    return vitrine\n\n\ndef triple_date_cats(vitrine: pd.DataFrame) -> pd.DataFrame:\n    preSeason = vitrine.query(\"preSeasonStartDate <= date <= preSeasonEndDate\").index\n    regularSeason = vitrine.query(\n        \"regularSeasonStartDate <= date <= regularSeasonEndDate\"\n    ).index\n    postSeason = vitrine.query(\"postSeasonStartDate <= date <= postSeasonEndDate\").index\n    allStarDate = vitrine.query(\"date == allStarDate\").index\n\n    vitrine.loc[:, \"triple_dates_cats\"] = 0\n    vitrine.loc[preSeason, \"triple_dates_cats\"] = 1\n    vitrine.loc[regularSeason, \"triple_dates_cats\"] = 2\n    vitrine.loc[postSeason, \"triple_dates_cats\"] = 3\n    vitrine.loc[allStarDate, \"triple_dates_cats\"] = 4\n\n    return vitrine\n\n\ndef triple_date_cats_2(vitrine: pd.DataFrame) -> pd.DataFrame:\n    preSeason = vitrine.query(\"preSeasonStartDate <= date <= preSeasonEndDate\").index\n    regularSeason_1 = vitrine.query(\n        \"regularSeasonStartDate <= date <= lastDate1stHalf\"\n    ).index\n    regularSeason_2 = vitrine.query(\n        \"firstDate2ndHalf <= date <= regularSeasonEndDate\"\n    ).index\n    postSeason = vitrine.query(\"postSeasonStartDate <= date <= postSeasonEndDate\").index\n    allStarDate = vitrine.query(\"date == allStarDate\").index\n\n    vitrine.loc[:, \"triple_dates_cats_2\"] = 0\n    vitrine.loc[preSeason, \"triple_dates_cats_2\"] = 1\n    vitrine.loc[regularSeason_1, \"triple_dates_cats_2\"] = 2\n    vitrine.loc[regularSeason_2, \"triple_dates_cats_2\"] = 3\n    vitrine.loc[postSeason, \"triple_dates_cats_2\"] = 4\n    vitrine.loc[allStarDate, \"triple_dates_cats_2\"] = 5\n\n    return vitrine\n\n\ndef double_date_cats(vitrine: pd.DataFrame) -> pd.DataFrame:\n    vitrine_preSeason = vitrine.query(\n        \"preSeasonStartDate <= date <= preSeasonEndDate\"\n    ).index\n    vitrine_Season = vitrine.query(\"seasonStartDate <= date <= seasonEndDate\").index\n    allStarDate = vitrine.query(\"date == allStarDate\").index\n\n    vitrine.loc[:, \"double_dates_cats\"] = 0\n    vitrine.loc[vitrine_preSeason, \"double_dates_cats\"] = 1\n    vitrine.loc[vitrine_Season, \"double_dates_cats\"] = 2\n    vitrine.loc[allStarDate, \"double_dates_cats\"] = 3\n\n    return vitrine\n\n\ndef double_date_cats_2(vitrine: pd.DataFrame) -> pd.DataFrame:\n    vitrine_preSeason = vitrine.query(\n        \"preSeasonStartDate <= date <= preSeasonEndDate\"\n    ).index\n    vitrine_Season_1 = vitrine.query(\"seasonStartDate <= date <= lastDate1stHalf\").index\n    vitrine_Season_2 = vitrine.query(\"firstDate2ndHalf <= date <= seasonEndDate\").index\n    allStarDate = vitrine.query(\"date == allStarDate\").index\n\n    vitrine.loc[:, \"double_dates_cats_2\"] = 0\n    vitrine.loc[vitrine_preSeason, \"double_dates_cats_2\"] = 1\n    vitrine.loc[vitrine_Season_1, \"double_dates_cats_2\"] = 2\n    vitrine.loc[vitrine_Season_2, \"double_dates_cats_2\"] = 3\n    vitrine.loc[allStarDate, \"double_dates_cats_2\"] = 4\n\n    return vitrine\n\n\ndef add_seasons_features(df: pd.DataFrame) -> pd.DataFrame:\n    df = triple_date_cats(df)\n    df = triple_date_cats_2(df)\n    df = double_date_cats(df)\n    df = double_date_cats_2(df)\n\n    return df.drop(\n        [\n            \"season\",\n            \"seasonStartDate\",\n            \"seasonEndDate\",\n            \"preSeasonStartDate\",\n            \"preSeasonEndDate\",\n            \"regularSeasonStartDate\",\n            \"regularSeasonEndDate\",\n            \"lastDate1stHalf\",\n            \"allStarDate\",\n            \"firstDate2ndHalf\",\n            \"postSeasonStartDate\",\n            \"postSeasonEndDate\",\n        ],\n        axis=1,\n    )\n\n\ndef generate_seasons(vitrine_i: pd.DataFrame) -> pd.DataFrame:\n    seasons = sns\n    vitrine = vitrine_i.copy()\n\n    vitrine = vitrine.loc[:, [\"id\", \"date\", \"game_id\", \"team_id\"]]\n    vitrine.loc[:, \"season\"] = vitrine[\"date\"].apply(lambda x: int(x[:4]))\n\n    vitrine = pd.merge(vitrine, seasons, on=[\"season\"])\n    vitrine = vitrine.replace([np.inf, -np.inf], np.nan)\n    vitrine = vitrine.fillna(-1)\n\n    vitrine = add_seasons_features(vitrine)\n    return vitrine\n\n\ndef generate_teams(vitrine: pd.DataFrame) -> pd.DataFrame:\n    teams = tms.copy()\n    types = teams.dtypes.to_dict()\n    del types[\"team_id\"]\n    \n    vitrine = pd.merge(\n        vitrine[[\"id\", \"date\", \"game_id\", \"team_id\"]], teams, on=[\"team_id\"], how=\"left\"\n    )\n    vitrine = vitrine.replace([np.inf, -np.inf], np.nan)\n    vitrine = vitrine.fillna(-1)\n    vitrine = vitrine.astype(types)\n    vitrine = decrease_mem_consuming(vitrine)\n\n    return vitrine\n\n\n\ndef day_before(val):\n    datetime_object = datetime.strptime(val, '%Y-%m-%d')\n    date_before = datetime_object - timedelta(1)\n    return date_before.strftime('%Y-%m-%d')\n\ndef get_base(df: pd.DataFrame) -> pd.DataFrame:\n    df[\"date\"] = df[\"date_playerId\"].apply(\n        lambda x: day_before(datetime.strftime(datetime.strptime(x.split(\"_\")[0], '%Y%m%d'), '%Y-%m-%d')))\n    df[\"id\"] = df[\"date_playerId\"].apply(lambda x: int( x.split(\"_\")[1]))\n    return df\n\n\ndef get_features(df, train):\n    df = get_base(df)\n    df = df\\\n    .drop([\"target1\",\"target2\",\"target3\",\"target4\"], axis=1)\\\n    .reset_index(drop=True)\n    base = df[['id', 'date']]\n    rosters = generate_rosters(train, base)\n    #print(rosters.shape)\n    games = generate_games(train, rosters[['id', 'date', 'team_id']])\n    #print(games.shape)\n    player_box = generate_player_box(train, games[['id', 'date', 'team_id', 'game_id']])\n    #print(player_box.shape)\n    team_box = generate_team_box(train, player_box[['id', 'date', 'team_id', 'game_id']])\n    #print(team_box.shape)\n    trans = generate_trans(train, team_box[['id', 'date', 'team_id', 'game_id']])\n    #print(trans.shape)\n    standings = generate_standings(train, trans[['id', 'date', 'team_id', 'game_id']])\n    #print(standings.shape)\n    awards = generate_awards(train, standings[['id', 'date', 'team_id', 'game_id']])\n    #print(awards.shape)\n    \n    \n    \n    player_twit = generate_player_twit(train, awards[['id', 'date', 'team_id', 'game_id']])\n    team_twit = generate_team_twit(train, awards[['id', 'date', 'team_id', 'game_id']])\n    \n    \n    players = generate_players(awards[['id', 'date', 'team_id', 'game_id']])\n    #print(players.shape)\n    seasons = generate_seasons(awards[['id', 'date', 'team_id', 'game_id']])\n    #print(seasons.shape)\n    teams = generate_teams(awards[['id', 'date', 'team_id', 'game_id']])\n    #print(teams.shape)\n    \n    \n    #print(rosters.columns)\n    #print(games.columns)\n    #print(player_box.columns)\n    #print(team_box.columns)\n    #print(trans.columns)\n    #print(standings.columns)\n    #print(awards.columns)\n    #print(player_twit)\n    #print(team_twit)\n    #print(players.columns)\n    #print(teams.columns)\n    \n    ret_1_ma_1 = np.array(\n        [0 for i in range(len(df))], dtype=np.float32)\n    ret_2_ma_1 = np.array(\n        [0 for i in range(len(df))], dtype=np.float32)\n    ret_3_ma_1 = np.array(\n        [0 for i in range(len(df))], dtype=np.float32)\n    ret_4_ma_1 = np.array(\n        [0 for i in range(len(df))], dtype=np.float32)\n    \n    \n    \n    for i, player_id in enumerate(df['id'].values):\n        lag_1 = d.get(player_id)\n        if lag_1 != None:\n            ret_1_ma_1[i] = lag_1['target_1_ma_med_100']\n            ret_2_ma_1[i] = lag_1['target_2_ma_med_100']\n            ret_3_ma_1[i] = lag_1['target_3_ma_med_100']\n            ret_4_ma_1[i] = lag_1['target_4_ma_med_100']\n    \n    df['target1'] = ret_1_ma_1\n    df['target2'] = ret_2_ma_1\n    df['target3'] = ret_3_ma_1\n    df['target4'] = ret_4_ma_1\n    \n    #standings = generate_standings(train, games)\n    #print(standings.shape)\n    for right_df in [rosters, games, player_box, team_box, trans, standings, awards, players, seasons, teams]:\n        df = pd.merge(\n            df,\n            right_df,\n            left_on=['id', 'date'],\n            right_on=['id', 'date'],\n            how='left',\n            suffixes=(\"\", \"_xx\"),\n        )\n        drop_names = list(filter(lambda x: x.find(\"_xx\") != -1, df.columns))\n        df = df.drop(drop_names, axis=1)\n    df = df.fillna(-1)\n    \n    for target_name, val in models.items():\n        if target_name in ['target1']:\n            df[target_name] = val['model'].predict(df.loc[:, val['features']])\n    df.index = df.date\n    #return decrease_mem_consuming(df)\n    return df.loc[:, ['date_playerId'] + [f'target{i+1}' for i in range(4)]]","metadata":{"execution":{"iopub.status.busy":"2021-07-23T15:07:32.654233Z","iopub.execute_input":"2021-07-23T15:07:32.654542Z","iopub.status.idle":"2021-07-23T15:07:32.806956Z","shell.execute_reply.started":"2021-07-23T15:07:32.654513Z","shell.execute_reply":"2021-07-23T15:07:32.805483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    sample_prediction_df = get_features(sample_prediction_df, test_df)\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T15:07:32.808366Z","iopub.execute_input":"2021-07-23T15:07:32.808672Z","iopub.status.idle":"2021-07-23T15:07:39.706761Z","shell.execute_reply.started":"2021-07-23T15:07:32.808642Z","shell.execute_reply":"2021-07-23T15:07:39.706048Z"},"trusted":true},"execution_count":null,"outputs":[]}]}