{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**This notebook is an exercise in the [Time Series](https://www.kaggle.com/learn/time-series) course.  You can reference the tutorial at [this link](https://www.kaggle.com/ryanholbrook/seasonality).**\n\n---\n","metadata":{}},{"cell_type":"markdown","source":"# Introduction #\n\nRun this cell to set everything up!","metadata":{}},{"cell_type":"code","source":"# Setup feedback system\nfrom learntools.core import binder\nbinder.bind(globals())\nfrom learntools.time_series.ex3 import *\n\n# Setup notebook\nfrom pathlib import Path\nfrom learntools.time_series.style import *  # plot style settings\nfrom learntools.time_series.utils import plot_periodogram, seasonal_plot\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nfrom sklearn.linear_model import LinearRegression\nfrom statsmodels.tsa.deterministic import CalendarFourier, DeterministicProcess\n\n\ncomp_dir = Path('../input/store-sales-time-series-forecasting')\n\nholidays_events = pd.read_csv(\n    comp_dir / \"holidays_events.csv\",\n    dtype={\n        'type': 'category',\n        'locale': 'category',\n        'locale_name': 'category',\n        'description': 'category',\n        'transferred': 'bool',\n    },\n    parse_dates=['date'],\n    infer_datetime_format=True,\n)\nholidays_events = holidays_events.set_index('date').to_period('D')\n\nstore_sales = pd.read_csv(\n    comp_dir / 'train.csv',\n    usecols=['store_nbr', 'family', 'date', 'sales'],\n    dtype={\n        'store_nbr': 'category',\n        'family': 'category',\n        'sales': 'float32',\n    },\n    parse_dates=['date'],\n    infer_datetime_format=True,\n)\nstore_sales['date'] = store_sales.date.dt.to_period('D')\nstore_sales = store_sales.set_index(['store_nbr', 'family', 'date']).sort_index()\naverage_sales = (\n    store_sales\n    .groupby('date').mean()\n    .squeeze()\n    .loc['2017']\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:47:43.333053Z","iopub.execute_input":"2022-07-14T14:47:43.334090Z","iopub.status.idle":"2022-07-14T14:47:51.282444Z","shell.execute_reply.started":"2022-07-14T14:47:43.333987Z","shell.execute_reply":"2022-07-14T14:47:51.281273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copie des fonctions définies dans le cours #\ndétail des fonctions de visualisation appelées ensuite","metadata":{}},{"cell_type":"code","source":"from warnings import simplefilter\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.linear_model import LinearRegression\nfrom statsmodels.tsa.deterministic import CalendarFourier, DeterministicProcess\n\nsimplefilter(\"ignore\")\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=16,\n    titlepad=10,\n)\nplot_params = dict(\n    color=\"0.75\",\n    style=\".-\",\n    markeredgecolor=\"0.25\",\n    markerfacecolor=\"0.25\",\n    legend=False,\n)\n%config InlineBackend.figure_format = 'retina'\n\n\n# annotations: https://stackoverflow.com/a/49238256/5769929\ndef seasonal_plot(X, y, period, freq, ax=None):\n    if ax is None:\n        _, ax = plt.subplots()\n    palette = sns.color_palette(\"husl\", n_colors=X[period].nunique(),)\n    ax = sns.lineplot(\n        x=freq,\n        y=y,\n        hue=period,\n        data=X,\n        ci=False,\n        ax=ax,\n        palette=palette,\n        legend=False,\n    )\n    ax.set_title(f\"Seasonal Plot ({period}/{freq})\")\n    for line, name in zip(ax.lines, X[period].unique()):\n        y_ = line.get_ydata()[-1]\n        ax.annotate(\n            name,\n            xy=(1, y_),\n            xytext=(6, 0),\n            color=line.get_color(),\n            xycoords=ax.get_yaxis_transform(),\n            textcoords=\"offset points\",\n            size=14,\n            va=\"center\",\n        )\n    return ax\n\n\ndef plot_periodogram(ts, detrend='linear', ax=None):\n    from scipy.signal import periodogram\n    fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\")\n    freqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\n    if ax is None:\n        _, ax = plt.subplots()\n    ax.step(freqencies, spectrum, color=\"purple\")\n    ax.set_xscale(\"log\")\n    ax.set_xticks([1, 2, 4, 6, 12, 26, 52, 104])\n    ax.set_xticklabels(\n        [\n            \"Annual (1)\",\n            \"Semiannual (2)\",\n            \"Quarterly (4)\",\n            \"Bimonthly (6)\",\n            \"Monthly (12)\",\n            \"Biweekly (26)\",\n            \"Weekly (52)\",\n            \"Semiweekly (104)\",\n        ],\n        rotation=30,\n    )\n    ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n    ax.set_ylabel(\"Variance\")\n    ax.set_title(\"Periodogram\")\n    return ax\n\n\ndata_dir = Path(\"../input/ts-course-data\")\ntunnel = pd.read_csv(data_dir / \"tunnel.csv\", parse_dates=[\"Day\"])\ntunnel = tunnel.set_index(\"Day\").to_period(\"D\")","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:47:51.284375Z","iopub.execute_input":"2022-07-14T14:47:51.284718Z","iopub.status.idle":"2022-07-14T14:47:51.343710Z","shell.execute_reply.started":"2022-07-14T14:47:51.284684Z","shell.execute_reply":"2022-07-14T14:47:51.342580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n\nExamine the following seasonal plot:","metadata":{}},{"cell_type":"code","source":"X = average_sales.to_frame()\nX[\"week\"] = X.index.week\nX[\"day\"] = X.index.dayofweek\nX.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:47:51.345232Z","iopub.execute_input":"2022-07-14T14:47:51.345704Z","iopub.status.idle":"2022-07-14T14:47:51.364244Z","shell.execute_reply.started":"2022-07-14T14:47:51.345667Z","shell.execute_reply":"2022-07-14T14:47:51.363452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.tail(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:47:55.805176Z","iopub.execute_input":"2022-07-14T14:47:55.805543Z","iopub.status.idle":"2022-07-14T14:47:55.816077Z","shell.execute_reply.started":"2022-07-14T14:47:55.805514Z","shell.execute_reply":"2022-07-14T14:47:55.814939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seasonal_plot(X, y='sales', period='week', freq='day');","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:47:56.724623Z","iopub.execute_input":"2022-07-14T14:47:56.725647Z","iopub.status.idle":"2022-07-14T14:47:57.555013Z","shell.execute_reply.started":"2022-07-14T14:47:56.725597Z","shell.execute_reply":"2022-07-14T14:47:57.554094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And also the periodogram:","metadata":{}},{"cell_type":"code","source":"average_sales.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:48:02.470944Z","iopub.execute_input":"2022-07-14T14:48:02.471844Z","iopub.status.idle":"2022-07-14T14:48:02.480579Z","shell.execute_reply.started":"2022-07-14T14:48:02.471804Z","shell.execute_reply":"2022-07-14T14:48:02.479821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_periodogram(average_sales);","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:48:03.755452Z","iopub.execute_input":"2022-07-14T14:48:03.756594Z","iopub.status.idle":"2022-07-14T14:48:04.286890Z","shell.execute_reply.started":"2022-07-14T14:48:03.756546Z","shell.execute_reply":"2022-07-14T14:48:04.285765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1) Determine seasonality\n\nWhat kind of seasonality do you see evidence of? Once you've thought about it, run the next cell for some discussion.","metadata":{}},{"cell_type":"code","source":"# View the solution (Run this cell to receive credit!)\nq_1.check()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:48:05.034663Z","iopub.execute_input":"2022-07-14T14:48:05.035273Z","iopub.status.idle":"2022-07-14T14:48:05.043378Z","shell.execute_reply.started":"2022-07-14T14:48:05.035238Z","shell.execute_reply":"2022-07-14T14:48:05.042489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n\n# 2) Create seasonal features\n\nUse `DeterministicProcess` and `CalendarFourier` to create:\n- indicators for weekly seasons and\n- Fourier features of order 4 for monthly seasons.","metadata":{}},{"cell_type":"code","source":"y = average_sales.copy()\n\n# Fourier features of order 4 for monthly seasons\nfourier = CalendarFourier(freq=\"M\", order=4)  # 4 sin/cos pairs for \"M\"onthly seasonality\n\ndp = DeterministicProcess(\n    index=y.index,\n    constant=True,\n    order=1,\n    # indicators for weekly seasons\n    seasonal=True,\n    # intégration de fourier\n    additional_terms=[fourier],  # monthly seasonality (fourier)\n    drop=True,\n)\nX = dp.in_sample()\n\n# Check your answer\nq_2.check()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:48:06.655218Z","iopub.execute_input":"2022-07-14T14:48:06.655873Z","iopub.status.idle":"2022-07-14T14:48:09.873457Z","shell.execute_reply.started":"2022-07-14T14:48:06.655836Z","shell.execute_reply":"2022-07-14T14:48:09.872322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:48:09.875792Z","iopub.execute_input":"2022-07-14T14:48:09.876470Z","iopub.status.idle":"2022-07-14T14:48:09.907549Z","shell.execute_reply.started":"2022-07-14T14:48:09.876426Z","shell.execute_reply":"2022-07-14T14:48:09.906359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:54:53.538668Z","iopub.execute_input":"2022-07-14T14:54:53.539367Z","iopub.status.idle":"2022-07-14T14:54:53.547028Z","shell.execute_reply.started":"2022-07-14T14:54:53.539320Z","shell.execute_reply":"2022-07-14T14:54:53.545864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:48:09.911335Z","iopub.execute_input":"2022-07-14T14:48:09.911774Z","iopub.status.idle":"2022-07-14T14:48:09.921573Z","shell.execute_reply.started":"2022-07-14T14:48:09.911732Z","shell.execute_reply":"2022-07-14T14:48:09.920264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now run this cell to fit the seasonal model.","metadata":{}},{"cell_type":"code","source":"model = LinearRegression().fit(X, y)\n\n#y_pred = pd.Series(\n#    model.predict(X),\n#    index=X.index,\n#    name='Fitted',\n#)\n\ny_pred1 = pd.Series(model.predict(X), index=X.index)\n\nax = y.plot(**plot_params, alpha=0.5, title=\"Average Sales\", ylabel=\"items sold\")\nax = y_pred1.plot(ax=ax, label=\"Seasonal_v1\")\nax.legend();","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:49:49.019689Z","iopub.execute_input":"2022-07-14T14:49:49.020513Z","iopub.status.idle":"2022-07-14T14:49:49.482183Z","shell.execute_reply.started":"2022-07-14T14:49:49.020471Z","shell.execute_reply":"2022-07-14T14:49:49.481025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n","metadata":{}},{"cell_type":"markdown","source":"Removing from a series its trend or seasons is called **detrending** or **deseasonalizing** the series.\n\nLook at the periodogram of the deseasonalized series.","metadata":{}},{"cell_type":"code","source":"y_deseason = y - y_pred1\n\nfig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, sharey=True, figsize=(10, 7))\nax1 = plot_periodogram(y, ax=ax1)\nax1.set_title(\"Product Sales Frequency Components\")\nax2 = plot_periodogram(y_deseason, ax=ax2);\nax2.set_title(\"Deseasonalized\");","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:47.885945Z","iopub.execute_input":"2022-07-14T14:50:47.886318Z","iopub.status.idle":"2022-07-14T14:50:48.631877Z","shell.execute_reply.started":"2022-07-14T14:50:47.886290Z","shell.execute_reply":"2022-07-14T14:50:48.631024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3) Check for remaining seasonality\n\nBased on these periodograms, how effectively does it appear your model captured the seasonality in *Average Sales*? Does the periodogram agree with the time plot of the deseasonalized series?","metadata":{}},{"cell_type":"code","source":"# View the solution (Run this cell to receive credit!)\nq_3.check()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:53.528519Z","iopub.execute_input":"2022-07-14T14:50:53.529661Z","iopub.status.idle":"2022-07-14T14:50:53.538746Z","shell.execute_reply.started":"2022-07-14T14:50:53.529616Z","shell.execute_reply":"2022-07-14T14:50:53.537603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n\nThe *Store Sales* dataset includes a table of Ecuadorian holidays.","metadata":{}},{"cell_type":"code","source":"# National and regional holidays in the training set\nholidays = (\n    holidays_events\n    .query(\"locale in ['National', 'Regional']\")\n    .loc['2017':'2017-08-15', ['description']]\n    .assign(description=lambda x: x.description.cat.remove_unused_categories())\n)\n\ndisplay(holidays)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:55.224803Z","iopub.execute_input":"2022-07-14T14:50:55.226008Z","iopub.status.idle":"2022-07-14T14:50:55.245267Z","shell.execute_reply.started":"2022-07-14T14:50:55.225929Z","shell.execute_reply":"2022-07-14T14:50:55.244035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From a plot of the deseasonalized *Average Sales*, it appears these holidays could have some predictive power.","metadata":{}},{"cell_type":"code","source":"ax = y_deseason.plot(**plot_params)\nplt.plot_date(holidays.index, y_deseason[holidays.index], color='C3')\nax.set_title('National and Regional Holidays');","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:59.467613Z","iopub.execute_input":"2022-07-14T14:50:59.468424Z","iopub.status.idle":"2022-07-14T14:50:59.859712Z","shell.execute_reply.started":"2022-07-14T14:50:59.468390Z","shell.execute_reply":"2022-07-14T14:50:59.858936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4) Create holiday features\n\nWhat kind of features could you create to help your model make use of this information? Code your answer in the next cell. (Scikit-learn and Pandas both have utilities that should make this easy. See the `hint` if you'd like more details.)\n","metadata":{}},{"cell_type":"code","source":"# YOUR CODE HERE\nX_holidays = pd.get_dummies(holidays)\nX_holidays.tail(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:10.522654Z","iopub.execute_input":"2022-07-14T14:51:10.523056Z","iopub.status.idle":"2022-07-14T14:51:10.540050Z","shell.execute_reply.started":"2022-07-14T14:51:10.523021Z","shell.execute_reply":"2022-07-14T14:51:10.539237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X2 = X.join(X_holidays, on='date').fillna(0.0)\nX2.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:11.734619Z","iopub.execute_input":"2022-07-14T14:51:11.735683Z","iopub.status.idle":"2022-07-14T14:51:11.771508Z","shell.execute_reply.started":"2022-07-14T14:51:11.735644Z","shell.execute_reply":"2022-07-14T14:51:11.770483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X2.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:15.847775Z","iopub.execute_input":"2022-07-14T14:51:15.848186Z","iopub.status.idle":"2022-07-14T14:51:15.856187Z","shell.execute_reply.started":"2022-07-14T14:51:15.848154Z","shell.execute_reply":"2022-07-14T14:51:15.855030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use this cell to fit the seasonal model with holiday features added. Do the fitted values seem to have improved?","metadata":{}},{"cell_type":"code","source":"model = LinearRegression().fit(X2, y)\n#y_pred = pd.Series(\n#    model.predict(X2),\n#    index=X2.index,\n#    name='Fitted',\n#)\n\ny_pred2 = pd.Series(model.predict(X2), index=X2.index)\n\nax = y.plot(**plot_params, alpha=0.5, title=\"Average Sales\", ylabel=\"items sold\")\nax = y_pred2.plot(ax=ax, label=\"Seasonal_v2\")\nax = y_pred1.plot(ax=ax, label=\"Seasonal_v1\", color=\"lavender\")\nax.legend();","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:40.784392Z","iopub.execute_input":"2022-07-14T14:52:40.784755Z","iopub.status.idle":"2022-07-14T14:52:41.276243Z","shell.execute_reply.started":"2022-07-14T14:52:40.784727Z","shell.execute_reply":"2022-07-14T14:52:41.275147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bonux : prédiction out of sample","metadata":{}},{"cell_type":"code","source":"X_fore = dp.out_of_sample(steps=90)\n# L'index est vide sur le out of sample\nX_fore.index.rename(\"date\", inplace=True)\n\n# Jointure avec les vacances dummy pour même nombre de colonnes que le dernier modèle entrainé\nX_fore = X_fore.join(X_holidays, on='date').fillna(0.0)\n\ny_fore = pd.Series(model.predict(X_fore), index=X_fore.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T15:00:15.183671Z","iopub.execute_input":"2022-07-14T15:00:15.184072Z","iopub.status.idle":"2022-07-14T15:00:15.201754Z","shell.execute_reply.started":"2022-07-14T15:00:15.184037Z","shell.execute_reply":"2022-07-14T15:00:15.200255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = y.plot(**plot_params, alpha=0.5, title=\"Average Sales + Forecast 90 days\", ylabel=\"items sold\")\nax = y_pred2.plot(ax=ax, label=\"Seasonal_v2\")\nax = y_fore.plot(ax=ax, label=\"Seasonal Forecast\", color='C3')\nax.legend();","metadata":{"execution":{"iopub.status.busy":"2022-07-14T15:01:44.885834Z","iopub.execute_input":"2022-07-14T15:01:44.886966Z","iopub.status.idle":"2022-07-14T15:01:45.445427Z","shell.execute_reply.started":"2022-07-14T15:01:44.886911Z","shell.execute_reply":"2022-07-14T15:01:45.444605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n\n# (Optional) Submit to Store Sales competition\n\nThis part of the exercise will walk you through your first submission to this course's companion competition: [**Store Sales - Time Series Forecasting**](https://www.kaggle.com/c/29781). Submitting to the competition isn't required to complete the course, but it's a great way to try out your new skills.\n\nThe next cell creates a seasonal model of the kind you've learned about in this lesson for the full *Store Sales* dataset with all 1800 time series.","metadata":{}},{"cell_type":"code","source":"y = store_sales.unstack(['store_nbr', 'family']).loc[\"2017\"]\n\n# Create training data\nfourier = CalendarFourier(freq='M', order=4)\ndp = DeterministicProcess(\n    index=y.index,\n    constant=True,\n    order=1,\n    seasonal=True,\n    additional_terms=[fourier],\n    drop=True,\n)\nX = dp.in_sample()\nX['NewYear'] = (X.index.dayofyear == 1)\n\nmodel = LinearRegression(fit_intercept=False)\nmodel.fit(X, y)\ny_pred = pd.DataFrame(model.predict(X), index=X.index, columns=y.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:34:29.369433Z","iopub.execute_input":"2022-07-14T14:34:29.370018Z","iopub.status.idle":"2022-07-14T14:34:33.911687Z","shell.execute_reply.started":"2022-07-14T14:34:29.369984Z","shell.execute_reply":"2022-07-14T14:34:33.899196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can use this cell to see some of its predictions.\n","metadata":{}},{"cell_type":"code","source":"STORE_NBR = '1'  # 1 - 54\nFAMILY = 'PRODUCE'\n# Uncomment to see a list of product families\n# display(store_sales.index.get_level_values('family').unique())\n\nax = y.loc(axis=1)['sales', STORE_NBR, FAMILY].plot(**plot_params)\nax = y_pred.loc(axis=1)['sales', STORE_NBR, FAMILY].plot(ax=ax)\nax.set_title(f'{FAMILY} Sales at Store {STORE_NBR}');","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:34:52.405542Z","iopub.execute_input":"2022-07-14T14:34:52.406561Z","iopub.status.idle":"2022-07-14T14:34:53.715986Z","shell.execute_reply.started":"2022-07-14T14:34:52.406521Z","shell.execute_reply":"2022-07-14T14:34:53.714502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, this cell loads the test data, creates a feature set for the forecast period, and then creates the submission file `submission.csv`.","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(\n    comp_dir / 'test.csv',\n    dtype={\n        'store_nbr': 'category',\n        'family': 'category',\n        'onpromotion': 'uint32',\n    },\n    parse_dates=['date'],\n    infer_datetime_format=True,\n)\ndf_test['date'] = df_test.date.dt.to_period('D')\ndf_test = df_test.set_index(['store_nbr', 'family', 'date']).sort_index()\n\n# Create features for test set\nX_test = dp.out_of_sample(steps=16)\nX_test.index.name = 'date'\nX_test['NewYear'] = (X_test.index.dayofyear == 1)\n\n\ny_submit = pd.DataFrame(model.predict(X_test), index=X_test.index, columns=y.columns)\ny_submit = y_submit.stack(['store_nbr', 'family'])\ny_submit = y_submit.join(df_test.id).reindex(columns=['id', 'sales'])\ny_submit.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:35:28.757278Z","iopub.execute_input":"2022-07-14T14:35:28.758428Z","iopub.status.idle":"2022-07-14T14:35:30.434350Z","shell.execute_reply.started":"2022-07-14T14:35:28.758396Z","shell.execute_reply":"2022-07-14T14:35:30.432667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To test your forecasts, you'll need to join the competition (if you haven't already). So open a new window by clicking on [this link](https://www.kaggle.com/c/29781). Then click on the **Join Competition** button.\n\nNext, follow the instructions below:\n1. Begin by clicking on the **Save Version** button in the top right corner of the window.  This will generate a pop-up window.  \n2. Ensure that the **Save and Run All** option is selected, and then click on the **Save** button.\n3. This generates a window in the bottom left corner of the notebook.  After it has finished running, click on the number to the right of the **Save Version** button.  This pulls up a list of versions on the right of the screen.  Click on the ellipsis **(...)** to the right of the most recent version, and select **Open in Viewer**.  This brings you into view mode of the same page. You will need to scroll down to get back to these instructions.\n4. Click on the **Output** tab on the right of the screen.  Then, click on the file you would like to submit, and click on the **Submit** button to submit your results to the leaderboard.\n\nYou have now successfully submitted to the competition!\n\nIf you want to keep working to improve your performance, select the **Edit** button in the top right of the screen. Then you can change your code and repeat the process. There's a lot of room to improve, and you will climb up the leaderboard as you work.\n","metadata":{}},{"cell_type":"markdown","source":"# Keep Going #\n\n[**Use time series as features**](https://www.kaggle.com/ryanholbrook/time-series-as-features) to capture cycles and other kinds of serial dependence.","metadata":{}},{"cell_type":"markdown","source":"---\n\n\n\n\n*Have questions or comments? Visit the [course discussion forum](https://www.kaggle.com/learn/time-series/discussion) to chat with other learners.*","metadata":{}}]}