{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport sys\nimport warnings\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\n\nimport ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom statsmodels.tsa.deterministic import (CalendarFourier,\n                                           CalendarSeasonality,\n                                           CalendarTimeTrend,\n                                           DeterministicProcess)\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers.experimental.preprocessing import StringLookup\n\nwarnings.simplefilter(\"ignore\")\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=14,\n    titlepad=10,\n)\nplot_params = dict(\n    color=\"0.75\",\n    style=\".-\",\n    markeredgecolor=\"0.25\",\n    markerfacecolor=\"0.25\",\n    legend=False,\n)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-19T08:19:19.807886Z","iopub.execute_input":"2021-06-19T08:19:19.808249Z","iopub.status.idle":"2021-06-19T08:19:25.500828Z","shell.execute_reply.started":"2021-06-19T08:19:19.808174Z","shell.execute_reply":"2021-06-19T08:19:25.499963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import all datasets","metadata":{}},{"cell_type":"code","source":"# Helper function to unpack json found in daily data\ndef unpack_json(json_str):\n    return pd.DataFrame() if pd.isna(json_str) else pd.read_json(json_str)\n\n\ndef unpack_data(data, dfs=None, n_jobs=-1):\n    if dfs is not None:\n        data = data.loc[:, dfs]\n    unnested_dfs = {}\n    for name, column in data.iteritems():\n        daily_dfs = Parallel(n_jobs=n_jobs)(\n            delayed(unpack_json)(item) for date, item in column.iteritems())\n        df = pd.concat(daily_dfs)\n        unnested_dfs[name] = df\n    return unnested_dfs","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:19:25.502201Z","iopub.execute_input":"2021-06-19T08:19:25.502529Z","iopub.status.idle":"2021-06-19T08:19:25.512045Z","shell.execute_reply.started":"2021-06-19T08:19:25.502495Z","shell.execute_reply":"2021-06-19T08:19:25.511150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Let's have a look at submission sample.","metadata":{}},{"cell_type":"code","source":"data_dir = Path('../input/mlb-player-digital-engagement-forecasting/')\n\ndf_names = ['seasons', 'teams', 'players', 'awards']\n\nfor name in df_names:\n    globals()[name] = pd.read_csv(data_dir / f\"{name}.csv\")\n\nkaggle_data_tabs = widgets.Tab()\n# Add Output widgets for each pandas DF as tabs' children\nkaggle_data_tabs.children = list([widgets.Output() for df_name in df_names])\n\nfor index in range(0, len(df_names)):\n    # Rename tab bar titles to df names\n    kaggle_data_tabs.set_title(index, df_names[index])\n    \n    # Display corresponding table output for this tab name\n    with kaggle_data_tabs.children[index]:\n        display(eval(df_names[index]))\n\ndisplay(kaggle_data_tabs)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:19:25.514599Z","iopub.execute_input":"2021-06-19T08:19:25.514887Z","iopub.status.idle":"2021-06-19T08:19:25.737956Z","shell.execute_reply.started":"2021-06-19T08:19:25.514857Z","shell.execute_reply":"2021-06-19T08:19:25.736968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Define dataframes to load from training set\ndfs = [\n    'nextDayPlayerEngagement',  # targets\n    'playerBoxScores',  # features\n    # Other dataframes available for features:\n    # 'games',\n    # 'rosters',\n    # 'teamBoxScores',\n    # 'transactions',\n    # 'standings',\n    # 'awards',\n    # 'events',\n    # 'playerTwitterFollowers',\n    # 'teamTwitterFollowers',\n]\n\n# Read training data\ntraining = pd.read_csv(\n    data_dir / 'train.csv',\n    usecols=['date'] + dfs,\n)\n\n# Convert training data date field to datetime type\ntraining['date'] = pd.to_datetime(training['date'], format=\"%Y%m%d\")\ntraining = training.set_index('date').to_period('D')\nprint(training.info())","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:19:25.739600Z","iopub.execute_input":"2021-06-19T08:19:25.740004Z","iopub.status.idle":"2021-06-19T08:20:19.318932Z","shell.execute_reply.started":"2021-06-19T08:19:25.739965Z","shell.execute_reply":"2021-06-19T08:20:19.317314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's also have a look at train and test datasets","metadata":{}},{"cell_type":"code","source":"%time\n# Unpack nested dataframes and store in dictionary `training_dfs`\ntraining_dfs = unpack_data(training, dfs=dfs)\nprint('\\n', training_dfs.keys())","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:20:19.320211Z","iopub.execute_input":"2021-06-19T08:20:19.320550Z","iopub.status.idle":"2021-06-19T08:20:57.604836Z","shell.execute_reply.started":"2021-06-19T08:20:19.320514Z","shell.execute_reply":"2021-06-19T08:20:57.603860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Players in the test set. We'll filter our data for only this set of players\npids_test = players.playerId.loc[\n    players.playerForTestSetAndFuturePreds.fillna(False)\n].astype(str)\n\n# Name of target columns\ntargets = [\"target1\", \"target2\", \"target3\", \"target4\"]\n\n\ndef make_playerBoxScores(dfs: dict, features):\n    X = dfs['playerBoxScores'].copy()\n    X = X[['gameDate', 'playerId'] + features]\n    # Set dtypes\n    X = X.astype({name: np.float32 for name in features})\n    X = X.astype({'playerId': str})\n    # Create date index\n    X = X.rename(columns={'gameDate': 'date'})\n    X['date'] = pd.PeriodIndex(X.date, freq='D')\n    # Aggregate multiple games per day by summing\n    X = X.groupby(['date', 'playerId'], as_index=False).sum()\n    return X\n\n\ndef make_targets(training_dfs: dict):\n    Y = training_dfs['nextDayPlayerEngagement'].copy()\n    # Set dtypes\n    Y = Y.astype({name: np.float32 for name in targets})\n    Y = Y.astype({'playerId': str})\n    # Match target dates to feature dates and create date index\n    Y = Y.rename(columns={'engagementMetricsDate': 'date'})\n    Y['date'] = pd.to_datetime(Y['date'])\n    Y = Y.set_index('date').to_period('D')\n    Y.index = Y.index - 1\n    return Y.reset_index()\n\n\ndef join_datasets(dfs):\n    dfs = [x.pivot(index='date', columns='playerId') for x in dfs]\n    df = pd.concat(dfs, axis=1).stack().reset_index('playerId')\n    return df\n\n\ndef make_training_data(training_dfs: dict,\n                       features,\n                       targets,\n                       fourier=4,\n                       test_size=30):\n    # Process dataframes\n    X = make_playerBoxScores(training_dfs, features)\n    Y = make_targets(training_dfs)\n    # Merge for processing\n    df = join_datasets([X, Y])\n    # Filter for players in test set\n    df = df.loc[df.playerId.isin(pids_test), :]\n    # Convert from long to wide format\n    df = df.pivot(columns=\"playerId\")\n    # Restore features and targets\n    X = df.loc(axis=1)[features, :]\n    Y = df.loc(axis=1)[targets, :]\n    # Fill missing values in features\n    X.fillna(-1, inplace=True)\n    # Create temporal features\n    fourier_terms = CalendarFourier(freq='A', order=fourier)\n    deterministic = DeterministicProcess(\n        index=X.index,\n        order=0,\n        seasonal=False,  # set to True for weekly seasonality\n        additional_terms=[fourier_terms],\n    )\n    X = pd.concat([X, deterministic.in_sample()], axis=1)\n    # Create train / validation splits\n    X_train, X_valid, y_train, y_valid = train_test_split(\n        X,\n        Y,\n        test_size=test_size,\n        shuffle=False,\n    )\n    return X_train, X_valid, y_train, y_valid, deterministic","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:20:57.606250Z","iopub.execute_input":"2021-06-19T08:20:57.606790Z","iopub.status.idle":"2021-06-19T08:20:57.625079Z","shell.execute_reply.started":"2021-06-19T08:20:57.606751Z","shell.execute_reply":"2021-06-19T08:20:57.624044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Columns to select from playerBoxScores, all numeric\nfeatures = [\n    \"hits\",\n    \"strikeOuts\",\n    \"homeRuns\",\n    \"runsScored\",\n    \"stolenBases\",\n    \"strikeOutsPitching\",\n    \"inningsPitched\",\n    \"strikes\",\n    \"flyOuts\",\n    \"groundOuts\",\n    \"errors\",\n]\n\n# Number of days to use for the validation set\ntest_size = 30\n\nX_train, X_valid, y_train, y_valid, deterministic = make_training_data(\n    training_dfs, \n    features=features, \n    targets=targets,\n    fourier=4,  # number of Fourier pairs describing annual seasonality\n    test_size=test_size,\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:20:57.626548Z","iopub.execute_input":"2021-06-19T08:20:57.626973Z","iopub.status.idle":"2021-06-19T08:21:19.584829Z","shell.execute_reply.started":"2021-06-19T08:20:57.626891Z","shell.execute_reply":"2021-06-19T08:21:19.583905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seasonal_plot(X, y, period, freq, ax=None):\n    if ax is None:\n        _, ax = plt.subplots()\n    palette = sns.color_palette(\n        \"husl\",\n        n_colors=X[period].nunique(),\n    )\n    ax = sns.lineplot(\n        x=freq,\n        y=y,\n        hue=period,\n        data=X,\n        ci=False,\n        ax=ax,\n        palette=palette,\n        legend=False,\n    )\n    ax.set_title(f\"Seasonal Plot ({period}/{freq})\")\n    for line, name in zip(ax.lines, X[period].unique()):\n        y_ = line.get_ydata()[-1]\n        ax.annotate(\n            name,\n            xy=(1, y_),\n            xytext=(6, 0),\n            color=line.get_color(),\n            xycoords=ax.get_yaxis_transform(),\n            textcoords=\"offset points\",\n            size=14,\n            va=\"center\",\n        )\n    return ax\n\n\ndef plot_periodogram(ts, detrend='linear', ax=None):\n    from scipy.signal import periodogram\n    fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\")\n    freqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\n    if ax is None:\n        _, ax = plt.subplots()\n    ax.step(freqencies, spectrum, color=\"purple\")\n    ax.set_xscale(\"log\")\n    ax.set_xticks([1, 2, 4, 6, 12, 26, 52, 104])\n    ax.set_xticklabels(\n        [\n            \"Annual\",\n            \"Semiannual\",\n            \"Quarterly\",\n            \"Bimonthly\",\n            \"Monthly\",\n            \"Biweekly\",\n            \"Weekly\",\n            \"Semiweekly\",\n        ],\n        rotation=30,\n    )\n    ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n    ax.set_ylabel(\"Density\")\n    ax.set_title(\"Periodogram\")\n    return ax","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:19.587557Z","iopub.execute_input":"2021-06-19T08:21:19.587908Z","iopub.status.idle":"2021-06-19T08:21:19.598837Z","shell.execute_reply.started":"2021-06-19T08:21:19.587865Z","shell.execute_reply":"2021-06-19T08:21:19.598007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select players in top decile of engagement\ndeciles = pd.qcut(y_train.mean().mean(level=1), q=10)\npids_top_decile = deciles.index[deciles == deciles.max()]\ny_top_decile = y_train.loc(axis=1)[:, pids_top_decile]\n\n# Create average engagement series\ny_top_decile_avg = (y_top_decile / y_top_decile.max(axis=0)).mean(axis=1)\ny_top_decile_avg.name = \"target\"\n\n# Yearly plot\nS = y_top_decile_avg.to_frame()\nS[\"month\"] = S.index.month  # the frequency\nS[\"year\"] = S.index.year  # the period\n_ = seasonal_plot(S, y=\"target\", period=\"year\", freq=\"month\")\n\n# Weekly plot\nS = y_top_decile_avg.to_frame()\nS[\"day\"] = S.index.dayofweek  # the frequency\nS[\"week\"] = S.index.week  # the period\n_ = seasonal_plot(S, y=\"target\", period=\"week\", freq=\"day\")","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:19.600408Z","iopub.execute_input":"2021-06-19T08:21:19.600902Z","iopub.status.idle":"2021-06-19T08:21:32.219904Z","shell.execute_reply.started":"2021-06-19T08:21:19.600858Z","shell.execute_reply":"2021-06-19T08:21:32.219086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plot_periodogram(y_top_decile_avg)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:32.221270Z","iopub.execute_input":"2021-06-19T08:21:32.221878Z","iopub.status.idle":"2021-06-19T08:21:32.679291Z","shell.execute_reply.started":"2021-06-19T08:21:32.221835Z","shell.execute_reply":"2021-06-19T08:21:32.678392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters\nHIDDEN = 1024\nACTIVATION = 'relu'  # could try elu, gelu, swish\nDROPOUT_RATE = 0.5\nLEARNING_RATE = 1e-2\nBATCH_SIZE = 32\n\nOUTPUTS = y_train.shape[-1]\nmodel = keras.Sequential([\n    layers.Dense(HIDDEN, activation=ACTIVATION),\n    layers.BatchNormalization(),\n    layers.Dropout(DROPOUT_RATE),\n    layers.Dense(HIDDEN, activation=ACTIVATION),\n    layers.BatchNormalization(),\n    layers.Dropout(DROPOUT_RATE),\n    layers.Dense(HIDDEN, activation=ACTIVATION),\n    layers.BatchNormalization(),\n    layers.Dropout(DROPOUT_RATE),\n    layers.Dense(OUTPUTS),\n])","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:32.680584Z","iopub.execute_input":"2021-06-19T08:21:32.680975Z","iopub.status.idle":"2021-06-19T08:21:34.640557Z","shell.execute_reply.started":"2021-06-19T08:21:32.680943Z","shell.execute_reply":"2021-06-19T08:21:34.639710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\noptimizer = keras.optimizers.Adam(learning_rate=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='mae', metrics=['mae'])\n\nearly_stopping = keras.callbacks.EarlyStopping(patience=3)\n\nhistory = model.fit(\n    X_train, y_train,\n    validation_data=(X_valid, y_valid),\n    batch_size=BATCH_SIZE,\n    epochs=100,\n    callbacks=[early_stopping],\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:34.641888Z","iopub.execute_input":"2021-06-19T08:21:34.642223Z","iopub.status.idle":"2021-06-19T08:21:41.526541Z","shell.execute_reply.started":"2021-06-19T08:21:34.642190Z","shell.execute_reply":"2021-06-19T08:21:41.525728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_test_data(test_dfs: dict, features, deterministic):\n    X = make_playerBoxScores(test_dfs, features)\n    X = X.merge(pids_test, how='right')\n    X['date'] = X.date.fillna(method='ffill').fillna(method='bfill')\n    X.fillna(-1, inplace=True)\n    # Convert from long to wide format\n    X = X.pivot(index='date', columns=\"playerId\")\n    # Create temporal features\n    X = pd.concat([\n        X,\n        deterministic.out_of_sample(steps=1, forecast_index=X.index),\n    ],\n                  axis=1)\n    return X\n\n\ndef make_predictions(model, X, columns, targets):\n    y_pred = model.predict(X)\n    y_pred = pd.DataFrame(y_pred, columns=columns, index=X.index).stack()\n    y_pred[targets] = y_pred[targets].clip(0, 100)\n    y_pred['date_playerId'] = [\n        (date + 1).strftime('%Y%m%d') + '_' + str(playerId)\n        for date, playerId in y_pred.index\n    ]\n    y_pred.reset_index('playerId', drop=True, inplace=True)\n    y_pred = y_pred[['date_playerId'] + targets]  # reorder\n    y_pred.index = pd.Int64Index(\n        [int(date.strftime('%Y%m%d')) for date in y_pred.index], name='date')\n    return y_pred","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:41.528050Z","iopub.execute_input":"2021-06-19T08:21:41.528394Z","iopub.status.idle":"2021-06-19T08:21:41.539092Z","shell.execute_reply.started":"2021-06-19T08:21:41.528357Z","shell.execute_reply":"2021-06-19T08:21:41.538175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport mlb\n\nenv = mlb.make_env()\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n    # Unpack features from test_df\n    test_dfs = unpack_data(test_df, dfs=['playerBoxScores'])\n    X = make_test_data(test_dfs, features, deterministic)\n\n    # Create predictions\n    y_pred = make_predictions(\n        model,\n        X,\n        columns=y_train.columns,\n        targets=targets,\n    )\n    submission = (\n        sample_prediction_df\n        [['date_playerId']]\n        .reset_index()  #  preserve index 'date'\n        .merge(y_pred, how='left', on='date_playerId')\n        .set_index('date')  #  restore index 'date'\n    )\n\n    # Submit predictions\n    env.predict(submission)  # constructs submissions.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-19T08:21:41.540511Z","iopub.execute_input":"2021-06-19T08:21:41.540898Z","iopub.status.idle":"2021-06-19T08:21:44.484818Z","shell.execute_reply.started":"2021-06-19T08:21:41.540860Z","shell.execute_reply":"2021-06-19T08:21:44.483777Z"},"trusted":true},"execution_count":null,"outputs":[]}]}