{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-01T04:02:57.270003Z","iopub.execute_input":"2021-07-01T04:02:57.270326Z","iopub.status.idle":"2021-07-01T04:02:57.278571Z","shell.execute_reply.started":"2021-07-01T04:02:57.270297Z","shell.execute_reply":"2021-07-01T04:02:57.277446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nimport ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport gc\nimport sys\nimport warnings\nfrom joblib import Parallel, delayed \n\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\n\nfrom statsmodels.tsa.deterministic import (CalendarFourier,    \n                                           CalendarSeasonality,\n                                           CalendarTimeTrend,\n                                           DeterministicProcess)\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers.experimental.preprocessing import StringLookup\nfrom datetime import timedelta\n\nwarnings.simplefilter(\"ignore\")\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=14,\n    titlepad=10,\n)\nplot_params = dict(\n    color=\"0.75\",\n    style=\".-\",\n    markeredgecolor=\"0.25\",\n    markerfacecolor=\"0.25\",\n    legend=False,\n)\nfeatures = [\n    \"hits\",\n    \"strikeOuts\",\n    \"homeRuns\",\n    \"runsScored\",\n    \"stolenBases\",\n    \"strikeOutsPitching\",\n    \"inningsPitched\",\n    \"strikes\",\n    \"flyOuts\",\n    \"groundOuts\",\n    \"errors\",\n]","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:02:57.330176Z","iopub.execute_input":"2021-07-01T04:02:57.330569Z","iopub.status.idle":"2021-07-01T04:03:04.310591Z","shell.execute_reply.started":"2021-07-01T04:02:57.330537Z","shell.execute_reply":"2021-07-01T04:03:04.309616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str):\n    return pd.DataFrame() if pd.isna(json_str) else pd.read_json(json_str)\n\n\ndef unpack_data(data, dfs=None, n_jobs=-1): #これか\n    if dfs is not None:\n        data = data.loc[:, dfs]\n    unnested_dfs = {}\n    for name, column in data.iteritems():\n        daily_dfs = Parallel(n_jobs=n_jobs)(\n            delayed(unpack_json)(item) for date, item in column.iteritems())\n        df = pd.concat(daily_dfs)\n        unnested_dfs[name] = df\n    return unnested_dfs","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:03:04.312363Z","iopub.execute_input":"2021-07-01T04:03:04.312748Z","iopub.status.idle":"2021-07-01T04:03:04.319363Z","shell.execute_reply.started":"2021-07-01T04:03:04.312709Z","shell.execute_reply":"2021-07-01T04:03:04.318315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path('../input/mlb-player-digital-engagement-forecasting/')\n\ndf_names = ['seasons', 'teams', 'players', 'awards']\n\nfor name in df_names:\n    globals()[name] = pd.read_csv(data_dir / f\"{name}.csv\")\n\nkaggle_data_tabs = widgets.Tab() #この指定も必要\n\nkaggle_data_tabs.children = list([widgets.Output() for df_name in df_names])  #>>>>>これを出力するとかっこいいリストになる\n\nfor index in range(0, len(df_names)):\n    # titleの名前を変える\n    kaggle_data_tabs.set_title(index, df_names[index])\n    \n    # Display corresponding table output for this tab name\n    with kaggle_data_tabs.children[index]:\n        display(eval(df_names[index]))\n\ndisplay(kaggle_data_tabs)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:03:04.321438Z","iopub.execute_input":"2021-07-01T04:03:04.322043Z","iopub.status.idle":"2021-07-01T04:03:04.519491Z","shell.execute_reply.started":"2021-07-01T04:03:04.322003Z","shell.execute_reply":"2021-07-01T04:03:04.518504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(data_dir / f\"awards.csv\")['awardName'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:03:04.520917Z","iopub.execute_input":"2021-07-01T04:03:04.521172Z","iopub.status.idle":"2021-07-01T04:03:04.550175Z","shell.execute_reply.started":"2021-07-01T04:03:04.521147Z","shell.execute_reply":"2021-07-01T04:03:04.549248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\noh = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/players.csv')\noh[oh['playerName']=='Shohei Ohtani']","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:03:04.551377Z","iopub.execute_input":"2021-07-01T04:03:04.551681Z","iopub.status.idle":"2021-07-01T04:03:04.575334Z","shell.execute_reply.started":"2021-07-01T04:03:04.551653Z","shell.execute_reply":"2021-07-01T04:03:04.574614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndfs = [\n    'nextDayPlayerEngagement', \n    'playerBoxScores',  \n   \n    'standings',\n    'playerTwitterFollowers',\n    'teamTwitterFollowers',\n]\n\ntraining = pd.read_csv(\n    data_dir / 'train.csv',\n    usecols=['date'] + dfs,\n)\n\ntraining['date'] = pd.to_datetime(training['date'], format=\"%Y%m%d\")\ntraining = training.set_index('date').to_period('D')\nprint(training.info())","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:03:04.576299Z","iopub.execute_input":"2021-07-01T04:03:04.576746Z","iopub.status.idle":"2021-07-01T04:04:00.359157Z","shell.execute_reply.started":"2021-07-01T04:03:04.576708Z","shell.execute_reply":"2021-07-01T04:04:00.358220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(data_dir / 'train.csv').head()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:04:00.360831Z","iopub.execute_input":"2021-07-01T04:04:00.361301Z","iopub.status.idle":"2021-07-01T04:04:48.797883Z","shell.execute_reply.started":"2021-07-01T04:04:00.361244Z","shell.execute_reply":"2021-07-01T04:04:48.796905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training[:7]","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:04:48.800207Z","iopub.execute_input":"2021-07-01T04:04:48.800525Z","iopub.status.idle":"2021-07-01T04:04:48.825791Z","shell.execute_reply.started":"2021-07-01T04:04:48.800497Z","shell.execute_reply":"2021-07-01T04:04:48.825044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(training['nextDayPlayerEngagement'][0])","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:04:48.827231Z","iopub.execute_input":"2021-07-01T04:04:48.827557Z","iopub.status.idle":"2021-07-01T04:04:48.838198Z","shell.execute_reply.started":"2021-07-01T04:04:48.827525Z","shell.execute_reply":"2021-07-01T04:04:48.837479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training['nextDayPlayerEngagement'][0][:1500]","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:04:48.839356Z","iopub.execute_input":"2021-07-01T04:04:48.839923Z","iopub.status.idle":"2021-07-01T04:04:48.849987Z","shell.execute_reply.started":"2021-07-01T04:04:48.839879Z","shell.execute_reply":"2021-07-01T04:04:48.849117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_dfs = unpack_data(training, dfs=dfs)\nprint('\\n', training_dfs.keys())","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:04:48.851458Z","iopub.execute_input":"2021-07-01T04:04:48.851746Z","iopub.status.idle":"2021-07-01T04:05:18.827338Z","shell.execute_reply.started":"2021-07-01T04:04:48.851716Z","shell.execute_reply":"2021-07-01T04:05:18.826339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_dfs","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:18.828599Z","iopub.execute_input":"2021-07-01T04:05:18.828891Z","iopub.status.idle":"2021-07-01T04:05:19.023525Z","shell.execute_reply.started":"2021-07-01T04:05:18.828858Z","shell.execute_reply":"2021-07-01T04:05:19.022544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_lag(df, lag=1):\n    dp = df[[\"playerId\",\"date\"]+TGTCOLS].copy()\n    dp[\"date\"]  =dp[\"date\"] + timedelta(days=lag) \n    df = df.merge(dp, on=[\"playerId\", \"date\"], suffixes=[\"\",f\"_{lag}\"], how=\"left\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:19.024754Z","iopub.execute_input":"2021-07-01T04:05:19.025049Z","iopub.status.idle":"2021-07-01T04:05:19.029831Z","shell.execute_reply.started":"2021-07-01T04:05:19.025019Z","shell.execute_reply":"2021-07-01T04:05:19.029112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pids_test = players.playerId.loc[\n    players.playerForTestSetAndFuturePreds.fillna(False)\n].astype(str)\n\n# Name of target columns\ntargets = [\"target1\", \"target2\", \"target3\", \"target4\"]\n\n\ndef make_playerBoxScores(dfs: dict, features):\n    X = dfs['playerBoxScores'].copy()\n    X = X[['gameDate', 'playerId'] + features]\n    # Set dtypes\n    X = X.astype({name: np.float32 for name in features})\n    X = X.astype({'playerId': str})\n    # Create date index\n    X = X.rename(columns={'gameDate': 'date'})\n    X['date'] = pd.PeriodIndex(X.date, freq='D')\n    # Aggregate multiple games per day by summing\n    X = X.groupby(['date', 'playerId'], as_index=False).sum()\n    return X\n\n\ndef make_targets(training_dfs: dict): \n    Y = training_dfs['nextDayPlayerEngagement'].copy()\n    # Set dtypes\n    Y = Y.astype({name: np.float32 for name in targets})\n    Y = Y.astype({'playerId': str})\n    # Match target dates to feature dates and create date index\n    Y = Y.rename(columns={'engagementMetricsDate': 'date'})\n    Y['date'] = pd.to_datetime(Y['date'])\n    Y = Y.set_index('date').to_period('D')\n    Y.index = Y.index - 1\n    return Y.reset_index()\n\n\ndef join_datasets(dfs):\n    dfs = [x.pivot(index='date', columns='playerId') for x in dfs]\n    df = pd.concat(dfs, axis=1).stack().reset_index('playerId')\n    return df\n\n\ndef make_training_data(training_dfs: dict,\n                       features,\n                       targets,\n                       fourier=4,\n                       test_size=30):\n    # Process dataframes\n    X = make_playerBoxScores(training_dfs, features)\n    Y = make_targets(training_dfs)\n    \n    \n    # Merge for processing\n    df = join_datasets([X, Y])\n    \n    \n    # Filter for players in test set\n    df = df.loc[df.playerId.isin(pids_test), :]\n    # Convert from long to wide format\n    df = df.pivot(columns=\"playerId\")\n    # Restore features and targets\n    X = df.loc(axis=1)[features, :]\n    Y = df.loc(axis=1)[targets, :]\n    # Fill missing values in features\n    X.fillna(-1, inplace=True)\n    # Create temporal features\n    fourier_terms = CalendarFourier(freq='A', order=fourier)\n    deterministic = DeterministicProcess(\n        index=X.index,\n        order=0,\n        seasonal=False,  # set to True for weekly seasonality\n        additional_terms=[fourier_terms],\n    )\n    X = pd.concat([X, deterministic.in_sample()], axis=1)\n    # Create train / validation splits\n    X_train, X_valid, y_train, y_valid = train_test_split(\n        X,\n        Y,\n        test_size=test_size,\n        shuffle=False,\n    )\n    return X_train, X_valid, y_train, y_valid, deterministic","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:19.030908Z","iopub.execute_input":"2021-07-01T04:05:19.031300Z","iopub.status.idle":"2021-07-01T04:05:19.049298Z","shell.execute_reply.started":"2021-07-01T04:05:19.031265Z","shell.execute_reply":"2021-07-01T04:05:19.048311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pids_test = players.playerId.loc[\n    players.playerForTestSetAndFuturePreds.fillna(False)\n].astype(str)\npids_test","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:19.050835Z","iopub.execute_input":"2021-07-01T04:05:19.051425Z","iopub.status.idle":"2021-07-01T04:05:19.071300Z","shell.execute_reply.started":"2021-07-01T04:05:19.051368Z","shell.execute_reply":"2021-07-01T04:05:19.070163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(players)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:19.072702Z","iopub.execute_input":"2021-07-01T04:05:19.073004Z","iopub.status.idle":"2021-07-01T04:05:19.080370Z","shell.execute_reply.started":"2021-07-01T04:05:19.072968Z","shell.execute_reply":"2021-07-01T04:05:19.079245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"players.playerForTestSetAndFuturePreds.isnull().sum() #.fillna(False)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:19.081783Z","iopub.execute_input":"2021-07-01T04:05:19.082075Z","iopub.status.idle":"2021-07-01T04:05:19.093651Z","shell.execute_reply.started":"2021-07-01T04:05:19.082044Z","shell.execute_reply":"2021-07-01T04:05:19.092715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X= make_playerBoxScores(training_dfs, features) \nX","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:19.094954Z","iopub.execute_input":"2021-07-01T04:05:19.095335Z","iopub.status.idle":"2021-07-01T04:05:33.692403Z","shell.execute_reply.started":"2021-07-01T04:05:19.095307Z","shell.execute_reply":"2021-07-01T04:05:33.691302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y =make_targets(training_dfs)\nY","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:33.694050Z","iopub.execute_input":"2021-07-01T04:05:33.694467Z","iopub.status.idle":"2021-07-01T04:05:37.115755Z","shell.execute_reply.started":"2021-07-01T04:05:33.694422Z","shell.execute_reply":"2021-07-01T04:05:37.114677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\nimport matplotlib.pyplot as plt\nplt.scatter(x = 'target1', y = 'target2',  data=Y)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:37.117354Z","iopub.execute_input":"2021-07-01T04:05:37.117769Z","iopub.status.idle":"2021-07-01T04:05:41.712634Z","shell.execute_reply.started":"2021-07-01T04:05:37.117727Z","shell.execute_reply":"2021-07-01T04:05:41.711781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\nimport matplotlib.pyplot as plt\nplt.scatter(x = 'target1', y = 'target3',  data=Y)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:41.713627Z","iopub.execute_input":"2021-07-01T04:05:41.713887Z","iopub.status.idle":"2021-07-01T04:05:46.448697Z","shell.execute_reply.started":"2021-07-01T04:05:41.713862Z","shell.execute_reply":"2021-07-01T04:05:46.447659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.scatter(x = 'target1', y = 'target4',  data=Y)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:46.450084Z","iopub.execute_input":"2021-07-01T04:05:46.450361Z","iopub.status.idle":"2021-07-01T04:05:50.980107Z","shell.execute_reply.started":"2021-07-01T04:05:46.450333Z","shell.execute_reply":"2021-07-01T04:05:50.979146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.scatter(x = 'target2', y = 'target3',  data=Y)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:50.985529Z","iopub.execute_input":"2021-07-01T04:05:50.985820Z","iopub.status.idle":"2021-07-01T04:05:55.547464Z","shell.execute_reply.started":"2021-07-01T04:05:50.985794Z","shell.execute_reply":"2021-07-01T04:05:55.546253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.heatmap(Y.corr(), vmin=-1.0, vmax=1.0, annot=True, cmap='coolwarm', linewidths=0.1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:55.550792Z","iopub.execute_input":"2021-07-01T04:05:55.551231Z","iopub.status.idle":"2021-07-01T04:05:56.090246Z","shell.execute_reply.started":"2021-07-01T04:05:55.551182Z","shell.execute_reply":"2021-07-01T04:05:56.088984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = join_datasets([X, Y])\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:56.091920Z","iopub.execute_input":"2021-07-01T04:05:56.092343Z","iopub.status.idle":"2021-07-01T04:05:59.769861Z","shell.execute_reply.started":"2021-07-01T04:05:56.092297Z","shell.execute_reply":"2021-07-01T04:05:59.768699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fourier=4\nfourier_terms = CalendarFourier(freq='A', order=fourier)\nfourier_terms","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:59.771428Z","iopub.execute_input":"2021-07-01T04:05:59.771841Z","iopub.status.idle":"2021-07-01T04:05:59.782607Z","shell.execute_reply.started":"2021-07-01T04:05:59.771798Z","shell.execute_reply":"2021-07-01T04:05:59.781602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pids_test ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:59.783894Z","iopub.execute_input":"2021-07-01T04:05:59.784186Z","iopub.status.idle":"2021-07-01T04:05:59.801326Z","shell.execute_reply.started":"2021-07-01T04:05:59.784155Z","shell.execute_reply":"2021-07-01T04:05:59.799944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.loc[df.playerId.isin(pids_test), :]\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:05:59.803137Z","iopub.execute_input":"2021-07-01T04:05:59.803536Z","iopub.status.idle":"2021-07-01T04:06:00.137532Z","shell.execute_reply.started":"2021-07-01T04:05:59.803501Z","shell.execute_reply":"2021-07-01T04:06:00.136521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.pivot(columns=\"playerId\")\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:00.139025Z","iopub.execute_input":"2021-07-01T04:06:00.139476Z","iopub.status.idle":"2021-07-01T04:06:00.975821Z","shell.execute_reply.started":"2021-07-01T04:06:00.139424Z","shell.execute_reply":"2021-07-01T04:06:00.974995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:00.976903Z","iopub.execute_input":"2021-07-01T04:06:00.977330Z","iopub.status.idle":"2021-07-01T04:06:00.982316Z","shell.execute_reply.started":"2021-07-01T04:06:00.977287Z","shell.execute_reply":"2021-07-01T04:06:00.981261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" X = df.loc(axis=1)[features, :]\n X","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:00.983710Z","iopub.execute_input":"2021-07-01T04:06:00.984174Z","iopub.status.idle":"2021-07-01T04:06:01.116402Z","shell.execute_reply.started":"2021-07-01T04:06:00.984134Z","shell.execute_reply":"2021-07-01T04:06:01.115419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = df.loc(axis=1)[targets, :]\nY","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.117607Z","iopub.execute_input":"2021-07-01T04:06:01.118063Z","iopub.status.idle":"2021-07-01T04:06:01.189764Z","shell.execute_reply.started":"2021-07-01T04:06:01.118015Z","shell.execute_reply":"2021-07-01T04:06:01.188745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = df.loc(axis=1)[features, '660271'].mean()\nr","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.191449Z","iopub.execute_input":"2021-07-01T04:06:01.191844Z","iopub.status.idle":"2021-07-01T04:06:01.211329Z","shell.execute_reply.started":"2021-07-01T04:06:01.191801Z","shell.execute_reply":"2021-07-01T04:06:01.210153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.fillna(-1, inplace=True)\ncheck = X.copy()\nX","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.212901Z","iopub.execute_input":"2021-07-01T04:06:01.213315Z","iopub.status.idle":"2021-07-01T04:06:01.646546Z","shell.execute_reply.started":"2021-07-01T04:06:01.213271Z","shell.execute_reply":"2021-07-01T04:06:01.645377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fourier_terms = CalendarFourier(freq='A', order=fourier)\nfourier_terms","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.648129Z","iopub.execute_input":"2021-07-01T04:06:01.648545Z","iopub.status.idle":"2021-07-01T04:06:01.655376Z","shell.execute_reply.started":"2021-07-01T04:06:01.648502Z","shell.execute_reply":"2021-07-01T04:06:01.654412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deterministic = DeterministicProcess(\n        index=X.index,\n        order=0,\n        seasonal=False,  # set to True for weekly seasonality\n        additional_terms=[fourier_terms],\n    )\ndeterministic","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.657234Z","iopub.execute_input":"2021-07-01T04:06:01.657731Z","iopub.status.idle":"2021-07-01T04:06:01.668164Z","shell.execute_reply.started":"2021-07-01T04:06:01.657634Z","shell.execute_reply":"2021-07-01T04:06:01.667227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in deterministic.in_sample():\n    print(i)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.669488Z","iopub.execute_input":"2021-07-01T04:06:01.669797Z","iopub.status.idle":"2021-07-01T04:06:01.697171Z","shell.execute_reply.started":"2021-07-01T04:06:01.669767Z","shell.execute_reply":"2021-07-01T04:06:01.696496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pd.concat([X, deterministic.in_sample()], axis=1)\nX","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.698090Z","iopub.execute_input":"2021-07-01T04:06:01.698325Z","iopub.status.idle":"2021-07-01T04:06:01.842917Z","shell.execute_reply.started":"2021-07-01T04:06:01.698301Z","shell.execute_reply":"2021-07-01T04:06:01.841956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" df.loc(axis=1)[features, :]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.844077Z","iopub.execute_input":"2021-07-01T04:06:01.844322Z","iopub.status.idle":"2021-07-01T04:06:01.967127Z","shell.execute_reply.started":"2021-07-01T04:06:01.844298Z","shell.execute_reply":"2021-07-01T04:06:01.966141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_size = 30\n\nX_train, X_valid, y_train, y_valid, deterministic = make_training_data(\n    training_dfs, \n    features=features, \n    targets=targets,\n    fourier=4,  \n    test_size=test_size,\n)\nX_train","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:01.968456Z","iopub.execute_input":"2021-07-01T04:06:01.968743Z","iopub.status.idle":"2021-07-01T04:06:24.916854Z","shell.execute_reply.started":"2021-07-01T04:06:01.968715Z","shell.execute_reply":"2021-07-01T04:06:24.915872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:24.918242Z","iopub.execute_input":"2021-07-01T04:06:24.918536Z","iopub.status.idle":"2021-07-01T04:06:24.959541Z","shell.execute_reply.started":"2021-07-01T04:06:24.918508Z","shell.execute_reply":"2021-07-01T04:06:24.958468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.mean()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:24.961169Z","iopub.execute_input":"2021-07-01T04:06:24.961614Z","iopub.status.idle":"2021-07-01T04:06:25.004367Z","shell.execute_reply.started":"2021-07-01T04:06:24.961571Z","shell.execute_reply":"2021-07-01T04:06:25.002815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.mean().mean(level=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.005975Z","iopub.execute_input":"2021-07-01T04:06:25.006452Z","iopub.status.idle":"2021-07-01T04:06:25.047957Z","shell.execute_reply.started":"2021-07-01T04:06:25.006402Z","shell.execute_reply":"2021-07-01T04:06:25.046848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.mean().mean(level=0)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.049477Z","iopub.execute_input":"2021-07-01T04:06:25.049825Z","iopub.status.idle":"2021-07-01T04:06:25.087776Z","shell.execute_reply.started":"2021-07-01T04:06:25.049790Z","shell.execute_reply":"2021-07-01T04:06:25.086623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r =y_train.loc(axis=1)[:,'660271']\nr\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.089642Z","iopub.execute_input":"2021-07-01T04:06:25.089955Z","iopub.status.idle":"2021-07-01T04:06:25.112483Z","shell.execute_reply.started":"2021-07-01T04:06:25.089926Z","shell.execute_reply":"2021-07-01T04:06:25.111298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check.loc(axis=1)[:,'660271'] ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.113908Z","iopub.execute_input":"2021-07-01T04:06:25.114233Z","iopub.status.idle":"2021-07-01T04:06:25.154786Z","shell.execute_reply.started":"2021-07-01T04:06:25.114201Z","shell.execute_reply":"2021-07-01T04:06:25.153700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r.mean()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.156025Z","iopub.execute_input":"2021-07-01T04:06:25.156291Z","iopub.status.idle":"2021-07-01T04:06:25.165460Z","shell.execute_reply.started":"2021-07-01T04:06:25.156265Z","shell.execute_reply":"2021-07-01T04:06:25.164518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r.mean().min(level=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.166616Z","iopub.execute_input":"2021-07-01T04:06:25.167022Z","iopub.status.idle":"2021-07-01T04:06:25.180247Z","shell.execute_reply.started":"2021-07-01T04:06:25.166993Z","shell.execute_reply":"2021-07-01T04:06:25.179182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.qcut(r.mean(), q=1) ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.181848Z","iopub.execute_input":"2021-07-01T04:06:25.182282Z","iopub.status.idle":"2021-07-01T04:06:25.212720Z","shell.execute_reply.started":"2021-07-01T04:06:25.182229Z","shell.execute_reply":"2021-07-01T04:06:25.211904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.qcut(r.mean(), q=2) ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.213785Z","iopub.execute_input":"2021-07-01T04:06:25.214187Z","iopub.status.idle":"2021-07-01T04:06:25.227044Z","shell.execute_reply.started":"2021-07-01T04:06:25.214156Z","shell.execute_reply":"2021-07-01T04:06:25.226017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.qcut(r.mean(), q=3) ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.228052Z","iopub.execute_input":"2021-07-01T04:06:25.228456Z","iopub.status.idle":"2021-07-01T04:06:25.240312Z","shell.execute_reply.started":"2021-07-01T04:06:25.228417Z","shell.execute_reply":"2021-07-01T04:06:25.239335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.mean().min(level=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.241616Z","iopub.execute_input":"2021-07-01T04:06:25.241904Z","iopub.status.idle":"2021-07-01T04:06:25.282684Z","shell.execute_reply.started":"2021-07-01T04:06:25.241876Z","shell.execute_reply":"2021-07-01T04:06:25.281522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.mean().max(level=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.284414Z","iopub.execute_input":"2021-07-01T04:06:25.284858Z","iopub.status.idle":"2021-07-01T04:06:25.325926Z","shell.execute_reply.started":"2021-07-01T04:06:25.284813Z","shell.execute_reply":"2021-07-01T04:06:25.324910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deciles = pd.qcut(y_train.mean().mean(level=1), q=5) \ndeciles","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.327459Z","iopub.execute_input":"2021-07-01T04:06:25.327852Z","iopub.status.idle":"2021-07-01T04:06:25.371721Z","shell.execute_reply.started":"2021-07-01T04:06:25.327809Z","shell.execute_reply":"2021-07-01T04:06:25.370598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pids_top_decile = deciles.index[deciles == deciles.max()] \npids_top_decile","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.373047Z","iopub.execute_input":"2021-07-01T04:06:25.373350Z","iopub.status.idle":"2021-07-01T04:06:25.382639Z","shell.execute_reply.started":"2021-07-01T04:06:25.373321Z","shell.execute_reply":"2021-07-01T04:06:25.381455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_top_decile = y_train.loc(axis=1)[:, pids_top_decile]\ny_top_decile","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.383876Z","iopub.execute_input":"2021-07-01T04:06:25.384199Z","iopub.status.idle":"2021-07-01T04:06:25.473650Z","shell.execute_reply.started":"2021-07-01T04:06:25.384170Z","shell.execute_reply":"2021-07-01T04:06:25.472539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_top_decile_avg = (y_top_decile / y_top_decile.max(axis=0)).mean(axis=1)\nS = y_top_decile_avg.to_frame()\nS.index.month","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.475313Z","iopub.execute_input":"2021-07-01T04:06:25.475704Z","iopub.status.idle":"2021-07-01T04:06:25.506175Z","shell.execute_reply.started":"2021-07-01T04:06:25.475662Z","shell.execute_reply":"2021-07-01T04:06:25.504970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\") #1年は何日か計算している\nfs","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.507914Z","iopub.execute_input":"2021-07-01T04:06:25.508224Z","iopub.status.idle":"2021-07-01T04:06:25.515819Z","shell.execute_reply.started":"2021-07-01T04:06:25.508191Z","shell.execute_reply":"2021-07-01T04:06:25.514249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import periodogram\nts =y_top_decile_avg\ndetrend='linear'\nax=None\nfreqencies, spectrum = periodogram(\n    ts,\n    fs=fs,\n    detrend=detrend,\n    window=\"boxcar\",\n    scaling='spectrum',\n)\nfreqencies[:20]","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.517639Z","iopub.execute_input":"2021-07-01T04:06:25.518078Z","iopub.status.idle":"2021-07-01T04:06:25.609422Z","shell.execute_reply.started":"2021-07-01T04:06:25.518029Z","shell.execute_reply":"2021-07-01T04:06:25.608587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_periodogram(ts, detrend='linear', ax=None):\n    from scipy.signal import periodogram\n    fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\")\n    freqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\n    if ax is None:\n        _, ax = plt.subplots()\n    ax.step(freqencies, spectrum, color=\"purple\") #これで線を引いているようだ\n    ax.set_xscale(\"log\")\n    ax.set_xticks([1, 2, 4, 6, 12, 26, 52, 104])\n    ax.set_xticklabels(\n        [\n            \"Annual\",\n            \"Semiannual\",\n            \"Quarterly\",\n            \"Bimonthly\",\n            \"Monthly\",\n            \"Biweekly\",\n            \"Weekly\",\n            \"Semiweekly\",\n        ],\n        rotation=30,\n    )\n    ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n    ax.set_ylabel(\"Density\")\n    ax.set_title(\"Periodogram\")\n    return ax\ndetrend='linear'\nfreqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\nfreqencies[:10]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.610850Z","iopub.execute_input":"2021-07-01T04:06:25.611364Z","iopub.status.idle":"2021-07-01T04:06:25.630170Z","shell.execute_reply.started":"2021-07-01T04:06:25.611331Z","shell.execute_reply":"2021-07-01T04:06:25.629047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrum[:11]","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.631948Z","iopub.execute_input":"2021-07-01T04:06:25.632645Z","iopub.status.idle":"2021-07-01T04:06:25.640979Z","shell.execute_reply.started":"2021-07-01T04:06:25.632597Z","shell.execute_reply":"2021-07-01T04:06:25.639920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plot_periodogram(y_top_decile_avg)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:25.642444Z","iopub.execute_input":"2021-07-01T04:06:25.642908Z","iopub.status.idle":"2021-07-01T04:06:26.072428Z","shell.execute_reply.started":"2021-07-01T04:06:25.642758Z","shell.execute_reply":"2021-07-01T04:06:26.071241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HIDDEN = 1024\nACTIVATION = 'relu' \nDROPOUT_RATE = 0.5\nLEARNING_RATE = 1e-2\nBATCH_SIZE = 32\n\nOUTPUTS = y_train.shape[-1]\nmodel = keras.Sequential([\n    layers.Dense(HIDDEN, activation=ACTIVATION),\n    layers.BatchNormalization(),\n    layers.Dropout(DROPOUT_RATE),\n    layers.Dense(HIDDEN, activation=ACTIVATION),\n    layers.BatchNormalization(),\n    layers.Dropout(DROPOUT_RATE),\n    layers.Dense(HIDDEN, activation=ACTIVATION),\n    layers.BatchNormalization(),\n    layers.Dropout(DROPOUT_RATE),\n    layers.Dense(OUTPUTS),\n])","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:26.074141Z","iopub.execute_input":"2021-07-01T04:06:26.074593Z","iopub.status.idle":"2021-07-01T04:06:26.164702Z","shell.execute_reply.started":"2021-07-01T04:06:26.074548Z","shell.execute_reply":"2021-07-01T04:06:26.163490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train1 = y_train['target1']\ny_train2 = y_train['target2']\ny_train3 = y_train['target3']\ny_train4 = y_train['target4']","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:26.166287Z","iopub.execute_input":"2021-07-01T04:06:26.166846Z","iopub.status.idle":"2021-07-01T04:06:26.175816Z","shell.execute_reply.started":"2021-07-01T04:06:26.166804Z","shell.execute_reply":"2021-07-01T04:06:26.174506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUTS","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:26.177672Z","iopub.execute_input":"2021-07-01T04:06:26.178316Z","iopub.status.idle":"2021-07-01T04:06:26.184804Z","shell.execute_reply.started":"2021-07-01T04:06:26.178273Z","shell.execute_reply":"2021-07-01T04:06:26.183977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[:7]","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:26.186110Z","iopub.execute_input":"2021-07-01T04:06:26.186542Z","iopub.status.idle":"2021-07-01T04:06:26.226605Z","shell.execute_reply.started":"2021-07-01T04:06:26.186498Z","shell.execute_reply":"2021-07-01T04:06:26.225364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = keras.optimizers.Adam(learning_rate=LEARNING_RATE) #1e-2\n\nmodel.compile(optimizer=optimizer, loss='mae', metrics=['mae'])\n\nearly_stopping = keras.callbacks.EarlyStopping(patience=3)\n\nhistory = model.fit(\n    X_train, y_train,\n    validation_data=(X_valid, y_valid),\n    batch_size=BATCH_SIZE,\n    epochs=90,\n    callbacks=[early_stopping],\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:06:26.228185Z","iopub.execute_input":"2021-07-01T04:06:26.228900Z","iopub.status.idle":"2021-07-01T04:07:18.583505Z","shell.execute_reply.started":"2021-07-01T04:06:26.228847Z","shell.execute_reply":"2021-07-01T04:07:18.582332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pids_test","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:18.588944Z","iopub.execute_input":"2021-07-01T04:07:18.589288Z","iopub.status.idle":"2021-07-01T04:07:18.597598Z","shell.execute_reply.started":"2021-07-01T04:07:18.589252Z","shell.execute_reply":"2021-07-01T04:07:18.596421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_test_data(test_dfs: dict, features, deterministic):\n    X = make_playerBoxScores(test_dfs, features)\n    X = X.merge(pids_test, how='right')\n    X['date'] = X.date.fillna(method='ffill').fillna(method='bfill')\n    X.fillna(-1, inplace=True)\n    # Convert from long to wide format\n    X = X.pivot(index='date', columns=\"playerId\")\n    # Create temporal features\n    X = pd.concat([\n        X,\n        deterministic.out_of_sample(steps=1, forecast_index=X.index),\n    ],\n                  axis=1)\n    return X","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:18.599304Z","iopub.execute_input":"2021-07-01T04:07:18.599591Z","iopub.status.idle":"2021-07-01T04:07:18.610094Z","shell.execute_reply.started":"2021-07-01T04:07:18.599564Z","shell.execute_reply":"2021-07-01T04:07:18.608806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_predictions(model, X, columns, targets):\n    y_pred = model.predict(X)\n    y_pred = pd.DataFrame(y_pred, columns=columns, index=X.index).stack()\n    y_pred[targets] = y_pred[targets].clip(0, 100)\n    y_pred['date_playerId'] = [\n        (date + 1).strftime('%Y%m%d') + '_' + str(playerId)\n        for date, playerId in y_pred.index\n    ]\n    y_pred.reset_index('playerId', drop=True, inplace=True)\n    y_pred = y_pred[['date_playerId'] + targets]  # reorder\n    y_pred.index = pd.Int64Index(\n        [int(date.strftime('%Y%m%d')) for date in y_pred.index], name='date')\n    return y_pred\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:18.611763Z","iopub.execute_input":"2021-07-01T04:07:18.612243Z","iopub.status.idle":"2021-07-01T04:07:18.623760Z","shell.execute_reply.started":"2021-07-01T04:07:18.612185Z","shell.execute_reply":"2021-07-01T04:07:18.622398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\n\nenv = mlb.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:18.625576Z","iopub.execute_input":"2021-07-01T04:07:18.626218Z","iopub.status.idle":"2021-07-01T04:07:18.661317Z","shell.execute_reply.started":"2021-07-01T04:07:18.626169Z","shell.execute_reply":"2021-07-01T04:07:18.660229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    # Unpack features from test_df\n    test_dfs = unpack_data(test_df, dfs=['playerBoxScores'])\n    X = make_test_data(test_dfs, features, deterministic)\n\n    # Create predictions\n    y_pred = make_predictions(\n        model,\n        X,\n        columns=y_train.columns,\n        targets=targets,\n    )\n    submission = (\n        sample_prediction_df\n        [['date_playerId']]\n        .reset_index()  #  preserve index 'date'\n        .merge(y_pred, how='left', on='date_playerId')\n        .set_index('date')  #  restore index 'date'\n    )\n\n    # Submit predictions\n    env.predict(submission)  # constructs submissions.csv","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:18.662573Z","iopub.execute_input":"2021-07-01T04:07:18.662842Z","iopub.status.idle":"2021-07-01T04:07:21.147203Z","shell.execute_reply.started":"2021-07-01T04:07:18.662815Z","shell.execute_reply":"2021-07-01T04:07:21.146144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dfs = unpack_data(test_df, dfs=['playerBoxScores'])\ntest_dfs","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:21.148526Z","iopub.execute_input":"2021-07-01T04:07:21.148833Z","iopub.status.idle":"2021-07-01T04:07:21.224028Z","shell.execute_reply.started":"2021-07-01T04:07:21.148801Z","shell.execute_reply":"2021-07-01T04:07:21.223015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = make_test_data(test_dfs, features, deterministic)\nX","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:21.225375Z","iopub.execute_input":"2021-07-01T04:07:21.225691Z","iopub.status.idle":"2021-07-01T04:07:21.323161Z","shell.execute_reply.started":"2021-07-01T04:07:21.225660Z","shell.execute_reply":"2021-07-01T04:07:21.322414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = make_predictions(\n        model,\n        X,\n        columns=y_train.columns,\n        targets=targets,\n    )\ny_pred","metadata":{"execution":{"iopub.status.busy":"2021-07-01T04:07:21.325458Z","iopub.execute_input":"2021-07-01T04:07:21.325904Z","iopub.status.idle":"2021-07-01T04:07:21.441323Z","shell.execute_reply.started":"2021-07-01T04:07:21.325857Z","shell.execute_reply":"2021-07-01T04:07:21.440189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}