{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom statsmodels.tsa.deterministic import (CalendarFourier,CalendarSeasonality,CalendarTimeTrend,DeterministicProcess)\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers.experimental.preprocessing import StringLookup\nimport gc\nimport sys\nimport warnings\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\n\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=14,\n    titlepad=10,\n)\nplot_params = dict(\n    color=\"0.75\",\n    style=\".-\",\n    markeredgecolor=\"0.25\",\n    markerfacecolor=\"0.25\",\n    legend=False,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T21:43:06.310029Z","iopub.execute_input":"2021-07-27T21:43:06.310466Z","iopub.status.idle":"2021-07-27T21:43:06.322288Z","shell.execute_reply.started":"2021-07-27T21:43:06.310420Z","shell.execute_reply":"2021-07-27T21:43:06.321157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter(\"ignore\")\n# Helper functions\ndef unpack_json(json_str):\n    return pd.DataFrame() if pd.isna(json_str) else pd.read_json(json_str)\n\n\ndef unpack_data(data, dfs=None, n_jobs=-1):\n    if dfs is not None:\n        data = data.loc[:, dfs]\n    unnested_dfs = {}\n    for name, column in data.iteritems():\n        daily_dfs = Parallel(n_jobs=n_jobs)(\n            delayed(unpack_json)(item) for date, item in column.iteritems())\n        df = pd.concat(daily_dfs)\n        unnested_dfs[name] = df\n    return unnested_dfs\n\ndef make_playerBoxScores(dfs: dict, features):\n    X = dfs['playerBoxScores'].copy()\n    X = X[['gameDate', 'playerId'] + features]\n    # Set dtypes\n    X = X.astype({name: np.float32 for name in features})\n    X = X.astype({'playerId': str})\n    # Create date index\n    X = X.rename(columns={'gameDate': 'date'})\n    X['date'] = pd.PeriodIndex(X.date, freq='D')\n    # Aggregate multiple games per day by summing\n    X = X.groupby(['date', 'playerId'], as_index=False).sum()\n    return X\n\n\ndef make_targets(training_dfs: dict):\n    Y = training_dfs['nextDayPlayerEngagement'].copy()\n    # Set dtypes\n    Y = Y.astype({name: np.float32 for name in targets})\n    Y = Y.astype({'playerId': str})\n    # Match target dates to feature dates and create date index\n    Y = Y.rename(columns={'engagementMetricsDate': 'date'})\n    Y['date'] = pd.to_datetime(Y['date'])\n    Y = Y.set_index('date').to_period('D')\n    Y.index = Y.index - 1\n    return Y.reset_index()\n\n\ndef join_datasets(dfs):\n    dfs = [x.pivot(index='date', columns='playerId') for x in dfs]\n    df = pd.concat(dfs, axis=1).stack().reset_index('playerId')\n    return df\n\n\ndef make_training_data(training_dfs: dict,\n                       features,\n                       targets,\n                       fourier=4,\n                       test_size=30):\n    # Process dataframes\n    X = make_playerBoxScores(training_dfs, features)\n    Y = make_targets(training_dfs)\n    # Merge for processing\n    df = join_datasets([X, Y])\n    # Filter for players in test set\n    df = df.loc[df.playerId.isin(pids_test), :]\n    # Convert from long to wide format\n    df = df.pivot(columns=\"playerId\")\n    # Restore features and targets\n    X = df.loc(axis=1)[features, :]\n    Y = df.loc(axis=1)[targets, :]\n    # Fill missing values in features\n    X.fillna(-1, inplace=True)\n    # Create temporal features\n    fourier_terms = CalendarFourier(freq='A', order=fourier)\n    deterministic = DeterministicProcess(\n        index=X.index,\n        order=0,\n        seasonal=False,  # set to True for weekly seasonality\n        additional_terms=[fourier_terms],\n    )\n    X = pd.concat([X, deterministic.in_sample()], axis=1)\n    # Create train / validation splits\n    X_train, X_valid, y_train, y_valid = train_test_split(\n        X,\n        Y,\n        test_size=test_size,\n        shuffle=False,\n    )\n    return X_train, X_valid, y_train, y_valid, deterministic\ndef seasonal_plot(X, y, period, freq, ax=None):\n    if ax is None:\n        _, ax = plt.subplots()\n    palette = sns.color_palette(\n        \"husl\",\n        n_colors=X[period].nunique(),\n    )\n    ax = sns.lineplot(\n        x=freq,\n        y=y,\n        hue=period,\n        data=X,\n        ci=False,\n        ax=ax,\n        palette=palette,\n        legend=False,\n    )\n    ax.set_title(f\"Seasonal Plot ({period}/{freq})\")\n    for line, name in zip(ax.lines, X[period].unique()):\n        y_ = line.get_ydata()[-1]\n        ax.annotate(\n            name,\n            xy=(1, y_),\n            xytext=(6, 0),\n            color=line.get_color(),\n            xycoords=ax.get_yaxis_transform(),\n            textcoords=\"offset points\",\n            size=14,\n            va=\"center\",\n        )\n    return ax\n\n\ndef plot_periodogram(ts, detrend='linear', ax=None):\n    from scipy.signal import periodogram\n    fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\")\n    freqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\n    if ax is None:\n        _, ax = plt.subplots()\n    ax.step(freqencies, spectrum, color=\"purple\")\n    ax.set_xscale(\"log\")\n    ax.set_xticks([1, 2, 4, 6, 12, 26, 52, 104])\n    ax.set_xticklabels(\n        [\n            \"Annual\",\n            \"Semiannual\",\n            \"Quarterly\",\n            \"Bimonthly\",\n            \"Monthly\",\n            \"Biweekly\",\n            \"Weekly\",\n            \"Semiweekly\",\n        ],\n        rotation=30,\n    )\n    ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n    ax.set_ylabel(\"Density\")\n    ax.set_title(\"Periodogram\")\n    return ax\n\ndef make_test_data(test_dfs: dict, features, deterministic):\n    X = make_playerBoxScores(test_dfs, features)\n    X = X.merge(pids_test, how='right')\n    X['date'] = X.date.fillna(method='ffill').fillna(method='bfill')\n    X.fillna(-1, inplace=True)\n    # Convert from long to wide format\n    X = X.pivot(index='date', columns=\"playerId\")\n    # Create temporal features\n    X = pd.concat([\n        X,\n        deterministic.out_of_sample(steps=1, forecast_index=X.index),\n    ],\n                  axis=1)\n    return X\n\n\ndef make_predictions(model, X, columns, targets):\n    y_pred = model.predict(X)\n    y_pred = pd.DataFrame(y_pred, columns=columns, index=X.index).stack()\n    y_pred[targets] = y_pred[targets].clip(0, 100)\n    y_pred['date_playerId'] = [\n        (date + 1).strftime('%Y%m%d') + '_' + str(playerId)\n        for date, playerId in y_pred.index\n    ]\n    y_pred.reset_index('playerId', drop=True, inplace=True)\n    y_pred = y_pred[['date_playerId'] + targets]  # reorder\n    y_pred.index = pd.Int64Index(\n        [int(date.strftime('%Y%m%d')) for date in y_pred.index], name='date')\n    return y_pred","metadata":{"execution":{"iopub.status.busy":"2021-07-27T21:43:06.324364Z","iopub.execute_input":"2021-07-27T21:43:06.324784Z","iopub.status.idle":"2021-07-27T21:43:06.365112Z","shell.execute_reply.started":"2021-07-27T21:43:06.324727Z","shell.execute_reply":"2021-07-27T21:43:06.364035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path('../input/mlb-player-digital-engagement-forecasting/')\n\ndf_names = ['seasons', 'teams', 'players', 'awards']\n\nfor name in df_names:\n    globals()[name] = pd.read_csv(data_dir / f\"{name}.csv\")\n\nkaggle_data_tabs = widgets.Tab()\n# Add Output widgets for each pandas DF as tabs' children\nkaggle_data_tabs.children = list([widgets.Output() for df_name in df_names])\n\nfor index in range(0, len(df_names)):\n    # Rename tab bar titles to df names\n    kaggle_data_tabs.set_title(index, df_names[index])\n    \n    # Display corresponding table output for this tab name\n    with kaggle_data_tabs.children[index]:\n        display(eval(df_names[index]))\n\ndisplay(kaggle_data_tabs)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.243274,"end_time":"2021-06-18T03:20:09.665119","exception":false,"start_time":"2021-06-18T03:20:09.421845","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T21:43:06.366920Z","iopub.execute_input":"2021-07-27T21:43:06.367341Z","iopub.status.idle":"2021-07-27T21:43:06.576198Z","shell.execute_reply.started":"2021-07-27T21:43:06.367309Z","shell.execute_reply":"2021-07-27T21:43:06.575256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [\n    'nextDayPlayerEngagement',  # targets\n    'playerBoxScores',  # features\n]\n\ntraining = pd.read_csv(\n    data_dir / 'train.csv',\n    usecols=['date'] + dfs,\n)\n\n# Convert training data date field to datetime type\ntraining['date'] = pd.to_datetime(training['date'], format=\"%Y%m%d\")\ntraining = training.set_index('date').to_period('D')\nprint(training.info())","metadata":{"_kg_hide-input":false,"papermill":{"duration":64.466592,"end_time":"2021-06-18T03:21:14.203064","exception":false,"start_time":"2021-06-18T03:20:09.736472","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T21:43:06.577607Z","iopub.execute_input":"2021-07-27T21:43:06.577915Z","iopub.status.idle":"2021-07-27T21:43:59.821906Z","shell.execute_reply.started":"2021-07-27T21:43:06.577884Z","shell.execute_reply":"2021-07-27T21:43:59.820803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unpack nested dataframes and store in dictionary `training_dfs`\ntraining_dfs = unpack_data(training, dfs=dfs)\nprint('\\n', training_dfs.keys())","metadata":{"papermill":{"duration":26.236092,"end_time":"2021-06-18T03:21:40.462615","exception":false,"start_time":"2021-06-18T03:21:14.226523","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T21:43:59.823470Z","iopub.execute_input":"2021-07-27T21:43:59.824105Z","iopub.status.idle":"2021-07-27T21:44:24.243339Z","shell.execute_reply.started":"2021-07-27T21:43:59.824040Z","shell.execute_reply":"2021-07-27T21:44:24.242216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Players in the test set. We'll filter our data for only this set of players\npids_test = players.playerId.loc[\n    players.playerForTestSetAndFuturePreds.fillna(False)\n].astype(str)\n\n# Name of target columns\ntargets = [\"target1\", \"target2\", \"target3\", \"target4\"]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.047912,"end_time":"2021-06-18T03:21:40.581553","exception":false,"start_time":"2021-06-18T03:21:40.533641","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T21:44:24.246318Z","iopub.execute_input":"2021-07-27T21:44:24.246730Z","iopub.status.idle":"2021-07-27T21:44:24.256458Z","shell.execute_reply.started":"2021-07-27T21:44:24.246697Z","shell.execute_reply":"2021-07-27T21:44:24.255430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    \"hits\",\n    \"strikeOuts\",\n    \"homeRuns\",\n    \"runsScored\",\n    \"stolenBases\",\n    \"strikeOutsPitching\",\n    \"inningsPitched\",\n    \"strikes\",\n    \"flyOuts\",\n    \"groundOuts\",\n    \"errors\",\n]\n\n# Number of days to use for the validation set\ntest_size = 30\n\n\nX_train, X_valid, y_train, y_valid, deterministic = make_training_data(\n    training_dfs, \n    features=features, \n    targets=targets,\n    fourier=4,  # number of Fourier pairs describing annual seasonality\n    test_size=test_size,\n)","metadata":{"papermill":{"duration":28.066727,"end_time":"2021-06-18T03:22:08.672119","exception":false,"start_time":"2021-06-18T03:21:40.605392","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T21:45:23.464435Z","iopub.execute_input":"2021-07-27T21:45:23.464845Z","iopub.status.idle":"2021-07-27T21:45:49.491083Z","shell.execute_reply.started":"2021-07-27T21:45:23.464809Z","shell.execute_reply":"2021-07-27T21:45:49.490027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deciles = pd.qcut(y_train.mean().mean(level=1), q=10)\npids_top_decile = deciles.index[deciles == deciles.max()]\ny_top_decile = y_train.loc(axis=1)[:, pids_top_decile]\n\n# Create average engagement series\ny_top_decile_avg = (y_top_decile / y_top_decile.max(axis=0)).mean(axis=1)\ny_top_decile_avg.name = \"target\"\n","metadata":{"execution":{"iopub.status.busy":"2021-07-27T21:45:49.494965Z","iopub.execute_input":"2021-07-27T21:45:49.495291Z","iopub.status.idle":"2021-07-27T21:45:49.571589Z","shell.execute_reply.started":"2021-07-27T21:45:49.495263Z","shell.execute_reply":"2021-07-27T21:45:49.570544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plot_periodogram(y_top_decile_avg)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.564637,"end_time":"2021-06-18T03:22:23.446789","exception":false,"start_time":"2021-06-18T03:22:22.882152","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-27T21:45:49.573198Z","iopub.execute_input":"2021-07-27T21:45:49.573550Z","iopub.status.idle":"2021-07-27T21:45:49.987752Z","shell.execute_reply.started":"2021-07-27T21:45:49.573518Z","shell.execute_reply":"2021-07-27T21:45:49.986754Z"},"trusted":true},"execution_count":null,"outputs":[]}]}