{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Final project for \"How to win a data science competition\" Coursera course","metadata":{}},{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"!pip install pmdarima","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:11.974891Z","iopub.execute_input":"2022-08-04T15:15:11.975783Z","iopub.status.idle":"2022-08-04T15:15:22.736757Z","shell.execute_reply.started":"2022-08-04T15:15:11.975726Z","shell.execute_reply":"2022-08-04T15:15:22.735532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\nimport statsmodels.api as sm\nimport pmdarima as pm\nfrom pmdarima.arima import auto_arima\nfrom statsmodels.graphics.tsaplots import plot_predict\n\n\nSTEPS = 31\nPERIODS = 4","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:22:15.643223Z","iopub.execute_input":"2022-08-04T15:22:15.643754Z","iopub.status.idle":"2022-08-04T15:22:15.654337Z","shell.execute_reply.started":"2022-08-04T15:22:15.643711Z","shell.execute_reply":"2022-08-04T15:22:15.652780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"markdown","source":"### File descriptions\n* sales_train.csv - the training set. Daily historical data from January 2013 to October 2015.\n* test.csv - the test set. You need to forecast the sales for these shops and products for November 2015.\n* sample_submission.csv - a sample submission file in the correct format.\n* items.csv - supplemental information about the items/products.\n* item_categories.csv  - supplemental information about the items categories.\n* shops.csv- supplemental information about the shops.","metadata":{}},{"cell_type":"code","source":"trainData = pd.read_csv('../input/competitive-data-science-predict-future-sales/sales_train.csv', parse_dates=['date'])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:22.756517Z","iopub.execute_input":"2022-08-04T15:15:22.756889Z","iopub.status.idle":"2022-08-04T15:15:24.522556Z","shell.execute_reply.started":"2022-08-04T15:15:22.756856Z","shell.execute_reply":"2022-08-04T15:15:24.521470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data fields\n* ID - an Id that represents a (Shop, Item) tuple within the test set\n* shop_id - unique identifier of a shop\n* item_id - unique identifier of a product\n* item_category_id - unique identifier of item category\n* item_cnt_day - number of products sold. You are predicting a monthly amount of this measure\n* item_price - current price of an item\n* date - date in format dd/mm/yyyy\n* date_block_num - a consecutive month number, used for convenience. January 2013 is 0, February 2013 is 1,..., October 2015 is 33\n* item_name - name of item\n* shop_name - name of shop\n* item_category_name - name of item category","metadata":{}},{"cell_type":"code","source":"trainData.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:24.525024Z","iopub.execute_input":"2022-08-04T15:15:24.525375Z","iopub.status.idle":"2022-08-04T15:15:24.539179Z","shell.execute_reply.started":"2022-08-04T15:15:24.525342Z","shell.execute_reply":"2022-08-04T15:15:24.538166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainData.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:24.540511Z","iopub.execute_input":"2022-08-04T15:15:24.541112Z","iopub.status.idle":"2022-08-04T15:15:24.976980Z","shell.execute_reply.started":"2022-08-04T15:15:24.541079Z","shell.execute_reply":"2022-08-04T15:15:24.975773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainData.hist(figsize=(20, 20))","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:24.978732Z","iopub.execute_input":"2022-08-04T15:15:24.979946Z","iopub.status.idle":"2022-08-04T15:15:26.622319Z","shell.execute_reply.started":"2022-08-04T15:15:24.979902Z","shell.execute_reply":"2022-08-04T15:15:26.621187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{}},{"cell_type":"code","source":"trainData = trainData[(trainData['item_price'] > 0) & (trainData['item_price'] < 10000)]\ntrainData = trainData[(trainData['item_cnt_day']>0) & (trainData['item_cnt_day'] < 1000)]","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:26.623506Z","iopub.execute_input":"2022-08-04T15:15:26.623838Z","iopub.status.idle":"2022-08-04T15:15:26.838601Z","shell.execute_reply.started":"2022-08-04T15:15:26.623807Z","shell.execute_reply":"2022-08-04T15:15:26.837454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(15, 8))\nsns.boxplot(trainData['item_price'], palette='PRGn', ax = ax[0, 0])\nsns.distplot(trainData['item_price'], ax = ax[1, 0])\nsns.boxplot(trainData['item_cnt_day'], palette='PRGn', ax = ax[0, 1])\nsns.distplot(trainData['item_cnt_day'], ax = ax[1, 1])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:26.839979Z","iopub.execute_input":"2022-08-04T15:15:26.840322Z","iopub.status.idle":"2022-08-04T15:15:50.227939Z","shell.execute_reply.started":"2022-08-04T15:15:26.840290Z","shell.execute_reply":"2022-08-04T15:15:50.226665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Time Series Analysis","metadata":{}},{"cell_type":"markdown","source":"### Rolling mean and standard deviation graphs","metadata":{}},{"cell_type":"code","source":"from statsmodels.tsa.stattools import adfuller\ndef test_stationarity(df, period = 12, dft=False):\n    movingAVG = df.rolling(window=period).mean()\n    movingSTD = df.rolling(window=period).std()\n    #plot\n    plt.figure(figsize=(15, 10))\n    df.plot(label='Original', alpha=0.3)\n    mean = plt.plot(movingAVG, label='Rolling Mean')\n    std = plt.plot(movingSTD, label='Rolling  Standard Deviation', alpha=0.5)\n    plt.title('Rolling Mean & Standard Deviation')\n    plt.legend()\n    if dft:\n        print('Results of Dickey-Fuller Test:')\n        dftest = adfuller(df, autolag='AIC')\n        dfoutput = pd.Series(dftest[0:4], index=['Test Statistic', 'p-value', '#Lags Used', 'Number of Observations Used'])\n        for key, value in dftest[4].items():\n            dfoutput['\\nCritical Value (%s)' % key] = value\n        print(dfoutput)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:50.229421Z","iopub.execute_input":"2022-08-04T15:15:50.229896Z","iopub.status.idle":"2022-08-04T15:15:50.249443Z","shell.execute_reply.started":"2022-08-04T15:15:50.229850Z","shell.execute_reply":"2022-08-04T15:15:50.240623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_stationarity(trainData['item_cnt_day'], period = 60)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:50.255519Z","iopub.execute_input":"2022-08-04T15:15:50.256370Z","iopub.status.idle":"2022-08-04T15:15:59.816049Z","shell.execute_reply.started":"2022-08-04T15:15:50.256318Z","shell.execute_reply":"2022-08-04T15:15:59.815167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Per month","metadata":{}},{"cell_type":"code","source":"trainData['Month'] = trainData['date'].dt.to_period('M')\ntrainData","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:15:59.817512Z","iopub.execute_input":"2022-08-04T15:15:59.818129Z","iopub.status.idle":"2022-08-04T15:16:00.153130Z","shell.execute_reply.started":"2022-08-04T15:15:59.818087Z","shell.execute_reply":"2022-08-04T15:16:00.151755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDataPerMonth = trainData.groupby(['Month']).agg({'item_cnt_day' : 'sum'})\ntrainDataPerMonth.reset_index(inplace=True)\ntrainDataPerMonth = trainDataPerMonth.set_index('Month')\ntrainDataPerMonth.rename(columns = {'item_cnt_day':'item_cnt_month'}, inplace = True)\ntrainDataPerMonth.plot()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:16:27.490501Z","iopub.execute_input":"2022-08-04T15:16:27.490912Z","iopub.status.idle":"2022-08-04T15:16:27.869794Z","shell.execute_reply.started":"2022-08-04T15:16:27.490877Z","shell.execute_reply":"2022-08-04T15:16:27.868629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Decomposition Type and data transformation","metadata":{}},{"cell_type":"code","source":"#import the required modules for TimeSeries data generation:\nimport statsmodels.api as sm\ndef tsdisplay(y, figsize = (15, 10), title = \"\", lags = 12):\n    tmp_data = y\n    fig = plt.figure(figsize = figsize)\n    #Plot the time series\n    tmp_data.plot(ax = fig.add_subplot(311), title = \"$Time\\ Series\\ \" + title + \"$\", legend = False)\n    #Plot the ACF:\n    sm.graphics.tsa.plot_acf(tmp_data, lags = lags, zero = False, ax = fig.add_subplot(323))\n    plt.xticks(np.arange(1,  lags + 1, 1.0))\n    #Plot the PACF:\n    sm.graphics.tsa.plot_pacf(tmp_data, lags = lags, zero = False, ax = fig.add_subplot(324))\n    plt.xticks(np.arange(1,  lags + 1, 1.0))\n    #Plot the QQ plot of the data:\n    sm.qqplot(tmp_data, line='s', ax = fig.add_subplot(325)) \n    plt.title(\"QQ Plot\")\n    #Plot the residual histogram:\n    fig.add_subplot(326).hist(tmp_data, bins = 40)\n    plt.title(\"Histogram\")\n    #Fix the layout of the plots:\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:16:00.524286Z","iopub.execute_input":"2022-08-04T15:16:00.524647Z","iopub.status.idle":"2022-08-04T15:16:00.534482Z","shell.execute_reply.started":"2022-08-04T15:16:00.524612Z","shell.execute_reply":"2022-08-04T15:16:00.533100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Import the required modules for model estimation:\nimport statsmodels.tsa as smt\ndef decomposition(df):\n    log_passengers = np.log(df)\n    decomposition_1  = smt.seasonal.seasonal_decompose(log_passengers, model = \"additive\", period = 12, two_sided = True)\n    fig, ax = plt.subplots(4, 1, figsize=(15, 8))\n    # Plot the series\n    decomposition_1.observed.plot(ax = ax[0])\n    decomposition_1.trend.plot(ax = ax[1])\n    decomposition_1.seasonal.plot(ax = ax[2])\n    decomposition_1.resid.plot(ax = ax[3])\n    # Add the labels to the Y-axis\n    ax[0].set_ylabel('Observed')\n    ax[1].set_ylabel('Trend')\n    ax[2].set_ylabel('Seasonal')\n    ax[3].set_ylabel('Residual')\n    # Fix layout\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:16:00.536323Z","iopub.execute_input":"2022-08-04T15:16:00.537273Z","iopub.status.idle":"2022-08-04T15:16:00.548499Z","shell.execute_reply.started":"2022-08-04T15:16:00.537217Z","shell.execute_reply":"2022-08-04T15:16:00.547737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decomposition(trainDataPerMonth)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:16:00.549842Z","iopub.execute_input":"2022-08-04T15:16:00.550158Z","iopub.status.idle":"2022-08-04T15:16:01.513729Z","shell.execute_reply.started":"2022-08-04T15:16:00.550127Z","shell.execute_reply":"2022-08-04T15:16:01.512485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" #### Checks for Stationarity of the time serie","metadata":{}},{"cell_type":"code","source":"test_stationarity(trainDataPerMonth['item_cnt_month'], 6, True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:16:01.515407Z","iopub.execute_input":"2022-08-04T15:16:01.515859Z","iopub.status.idle":"2022-08-04T15:16:01.918265Z","shell.execute_reply.started":"2022-08-04T15:16:01.515822Z","shell.execute_reply":"2022-08-04T15:16:01.917244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The p-value is greater than the critical value of 0.05. The series is not stationary","metadata":{}},{"cell_type":"markdown","source":"## Forecasting","metadata":{}},{"cell_type":"markdown","source":"### Train Test Split","metadata":{}},{"cell_type":"code","source":"log_passengers = np.log(trainDataPerMonth)\ntrainDataPerMonthShift = log_passengers - log_passengers.shift()\ntrainDataPerMonthShift.dropna(inplace=True)\ntrainDataPerMonthShift.plot()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:18:00.352402Z","iopub.execute_input":"2022-08-04T15:18:00.352916Z","iopub.status.idle":"2022-08-04T15:18:00.652544Z","shell.execute_reply.started":"2022-08-04T15:18:00.352878Z","shell.execute_reply":"2022-08-04T15:18:00.651746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = trainDataPerMonthShift[:STEPS]\ntest = trainDataPerMonthShift[STEPS:]\n\nprint(f\"Train dates : {train.index.min()} --- {train.index.max()}  (n={len(train)})\")\nprint(f\"Test dates  : {test.index.min()} --- {test.index.max()}  (n={len(test)})\")\n\nfig, ax = plt.subplots(figsize=(9, 4))\ntrain['item_cnt_month'].plot(ax=ax, label='train')\ntest['item_cnt_month'].plot(ax=ax, label='test')\nax.legend()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:18:03.994598Z","iopub.execute_input":"2022-08-04T15:18:03.995378Z","iopub.status.idle":"2022-08-04T15:18:04.326467Z","shell.execute_reply.started":"2022-08-04T15:18:03.995337Z","shell.execute_reply":"2022-08-04T15:18:04.325623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ARIMA","metadata":{}},{"cell_type":"code","source":"trainShift1 = train - train.shift(1)\ntrainShift1 = trainShift1.dropna()\nmodel = sm.tsa.arima.ARIMA(trainShift1, order=(0,0,2))\nresults_ARIMA = model.fit()\ntrain.plot()\nplt.plot(results_ARIMA.fittedvalues, color='red')\nplt.title('ARIMA model')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:21:32.691462Z","iopub.execute_input":"2022-08-04T15:21:32.692577Z","iopub.status.idle":"2022-08-04T15:21:33.343459Z","shell.execute_reply.started":"2022-08-04T15:21:32.692536Z","shell.execute_reply":"2022-08-04T15:21:33.342204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_predict(results_ARIMA, 1,36)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:22:27.257218Z","iopub.execute_input":"2022-08-04T15:22:27.257723Z","iopub.status.idle":"2022-08-04T15:22:27.989840Z","shell.execute_reply.started":"2022-08-04T15:22:27.257661Z","shell.execute_reply":"2022-08-04T15:22:27.988521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ARIMAForecast = results_ARIMA.forecast(steps=PERIODS)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:07.650295Z","iopub.execute_input":"2022-08-04T15:23:07.650790Z","iopub.status.idle":"2022-08-04T15:23:07.668731Z","shell.execute_reply.started":"2022-08-04T15:23:07.650754Z","shell.execute_reply":"2022-08-04T15:23:07.667149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom sklearn.metrics import mean_squared_error\nmean_squared_error(ARIMAForecast,test['item_cnt_month'])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:12.443743Z","iopub.execute_input":"2022-08-04T15:23:12.444164Z","iopub.status.idle":"2022-08-04T15:23:12.451513Z","shell.execute_reply.started":"2022-08-04T15:23:12.444131Z","shell.execute_reply":"2022-08-04T15:23:12.450750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = pd.Series(ARIMAForecast, index = test.index)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:14.859082Z","iopub.execute_input":"2022-08-04T15:23:14.859538Z","iopub.status.idle":"2022-08-04T15:23:14.866653Z","shell.execute_reply.started":"2022-08-04T15:23:14.859504Z","shell.execute_reply":"2022-08-04T15:23:14.865281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(9, 4))\ntrain['item_cnt_month'].plot(ax=ax, label='Train')\ntest['item_cnt_month'].plot(ax=ax, label='Test')\npred.plot(label='Forecast')\nax.legend()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:17.080507Z","iopub.execute_input":"2022-08-04T15:23:17.081927Z","iopub.status.idle":"2022-08-04T15:23:17.363174Z","shell.execute_reply.started":"2022-08-04T15:23:17.081882Z","shell.execute_reply":"2022-08-04T15:23:17.361747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Auto ARIMA + SHIFT","metadata":{}},{"cell_type":"code","source":"trainDataPerMonthDiff = trainDataPerMonth - trainDataPerMonth.shift()\ntrainDataPerMonthDiff = trainDataPerMonthDiff.dropna()\ntest_stationarity(trainDataPerMonthDiff['item_cnt_month'], 12, True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:21.117745Z","iopub.execute_input":"2022-08-04T15:23:21.118283Z","iopub.status.idle":"2022-08-04T15:23:21.501599Z","shell.execute_reply.started":"2022-08-04T15:23:21.118243Z","shell.execute_reply":"2022-08-04T15:23:21.500285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelWithShift = auto_arima(\n    y=trainDataPerMonthDiff[:STEPS],\n    seasonal=True,\n    start_p = 0, max_p =5,\n    start_q = 0, max_q =5,\n    d=None,\n    start_P = 1, max_P =5,\n    start_Q = 1, max_Q =5,\n    D=1,\n    m=12,\n    test = \"adf\",\n    trace = True, \n    information_criterion = 'aic', \n    suppress_warnings = True, \n    stepwise = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:24.249390Z","iopub.execute_input":"2022-08-04T15:23:24.249846Z","iopub.status.idle":"2022-08-04T15:23:31.180081Z","shell.execute_reply.started":"2022-08-04T15:23:24.249808Z","shell.execute_reply":"2022-08-04T15:23:31.176240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelWithShift.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:33.340462Z","iopub.execute_input":"2022-08-04T15:23:33.340887Z","iopub.status.idle":"2022-08-04T15:23:33.361875Z","shell.execute_reply.started":"2022-08-04T15:23:33.340853Z","shell.execute_reply":"2022-08-04T15:23:33.360678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelWithShift.plot_diagnostics(figsize=(14,10))","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:36.631055Z","iopub.execute_input":"2022-08-04T15:23:36.631443Z","iopub.status.idle":"2022-08-04T15:23:37.741175Z","shell.execute_reply.started":"2022-08-04T15:23:36.631413Z","shell.execute_reply":"2022-08-04T15:23:37.740063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction, confint = modelWithShift.predict(n_periods=PERIODS, return_conf_int=True)\nconfint_df = pd.DataFrame(confint)\nprediction","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:42.480990Z","iopub.execute_input":"2022-08-04T15:23:42.481407Z","iopub.status.idle":"2022-08-04T15:23:42.495373Z","shell.execute_reply.started":"2022-08-04T15:23:42.481374Z","shell.execute_reply":"2022-08-04T15:23:42.494049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period_index = pd.period_range(\n    start = trainDataPerMonthDiff[STEPS:].index[0],\n    periods = PERIODS,\n    freq='M'\n)\npredicted_df = pd.DataFrame({'value':prediction}, index=period_index)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:44.926763Z","iopub.execute_input":"2022-08-04T15:23:44.927179Z","iopub.status.idle":"2022-08-04T15:23:44.934359Z","shell.execute_reply.started":"2022-08-04T15:23:44.927144Z","shell.execute_reply":"2022-08-04T15:23:44.933200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.plot(trainDataPerMonthDiff[:STEPS].to_timestamp(), label='Actual')\nplt.plot(predicted_df.to_timestamp(), color='orange', label='Predicted')\nplt.plot(trainDataPerMonthDiff[STEPS:], label='Test')\nplt.fill_between(period_index.to_timestamp(), confint_df[0], confint_df[1], color='grey', alpha=.2, label='Confidence Intervals Area')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:48.484924Z","iopub.execute_input":"2022-08-04T15:23:48.485462Z","iopub.status.idle":"2022-08-04T15:23:48.806156Z","shell.execute_reply.started":"2022-08-04T15:23:48.485414Z","shell.execute_reply":"2022-08-04T15:23:48.805195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Auto ARIMA","metadata":{}},{"cell_type":"code","source":"model=auto_arima(trainDataPerMonth[:STEPS],\n                 start_p = 0, start_q = 0, \n                 d=1,\n                 D=1, \n                 m = 12, \n                 seasonal = True, \n                 test = \"adf\",  \n                 trace = True, \n                 alpha = 0.05, \n                 information_criterion = 'aic', \n                 suppress_warnings = True, \n                 stepwise = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:52.199711Z","iopub.execute_input":"2022-08-04T15:23:52.200140Z","iopub.status.idle":"2022-08-04T15:23:53.297343Z","shell.execute_reply.started":"2022-08-04T15:23:52.200105Z","shell.execute_reply":"2022-08-04T15:23:53.295820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:23:58.883981Z","iopub.execute_input":"2022-08-04T15:23:58.884422Z","iopub.status.idle":"2022-08-04T15:23:58.902507Z","shell.execute_reply.started":"2022-08-04T15:23:58.884385Z","shell.execute_reply":"2022-08-04T15:23:58.901403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.plot_diagnostics(figsize=(14,10))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:01.656461Z","iopub.execute_input":"2022-08-04T15:24:01.656878Z","iopub.status.idle":"2022-08-04T15:24:02.299594Z","shell.execute_reply.started":"2022-08-04T15:24:01.656845Z","shell.execute_reply":"2022-08-04T15:24:02.298671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction, confint = model.predict(n_periods = PERIODS, return_conf_int = True)\nperiod_index = pd.period_range(start = trainDataPerMonthDiff[STEPS:].index[0], periods = PERIODS, freq='M')\nforecast = pd.DataFrame({'Predicted item_cnt_month': prediction.round(2)}, index = period_index)\nforecast.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:18.504381Z","iopub.execute_input":"2022-08-04T15:24:18.504841Z","iopub.status.idle":"2022-08-04T15:24:18.524746Z","shell.execute_reply.started":"2022-08-04T15:24:18.504803Z","shell.execute_reply":"2022-08-04T15:24:18.523416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confint_df = pd.DataFrame(confint)\nprediction_series = pd.Series(prediction,index=period_index)\nplt.figure(figsize=(15, 10))\ntrainDataPerMonth['item_cnt_month'][:STEPS].plot(color='red', label='Actual')\nprediction_series.plot(color='orange', label='Predicted')\ntrainDataPerMonth['item_cnt_month'][STEPS:].plot(color='green', label='Test')\nplt.fill_between(prediction_series.index.to_timestamp(), confint_df[0], confint_df[1], color='grey', alpha=.2, label='Confidence Intervals Area')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:09.168745Z","iopub.execute_input":"2022-08-04T15:24:09.169211Z","iopub.status.idle":"2022-08-04T15:24:09.580914Z","shell.execute_reply.started":"2022-08-04T15:24:09.169176Z","shell.execute_reply":"2022-08-04T15:24:09.579661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF = trainData.groupby(['shop_id', 'item_id'])['date', 'item_cnt_day'].agg({'item_cnt_day':'sum'})\ntrainDF = trainDF.reset_index()\ntrainDF.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:23.094256Z","iopub.execute_input":"2022-08-04T15:24:23.094648Z","iopub.status.idle":"2022-08-04T15:24:23.409488Z","shell.execute_reply.started":"2022-08-04T15:24:23.094608Z","shell.execute_reply":"2022-08-04T15:24:23.408300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"markdown","source":"### Loading test data","metadata":{}},{"cell_type":"code","source":"testData = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/test.csv\")\ntestData.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:26.195285Z","iopub.execute_input":"2022-08-04T15:24:26.195729Z","iopub.status.idle":"2022-08-04T15:24:26.301186Z","shell.execute_reply.started":"2022-08-04T15:24:26.195675Z","shell.execute_reply":"2022-08-04T15:24:26.300023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testData['item_cnt_month'] = (prediction[0].round(2)*len(testData)/len(testData))/len(testData)\nsubmission  = testData.drop(['shop_id', 'item_id'], axis = 1)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:28.321905Z","iopub.execute_input":"2022-08-04T15:24:28.323042Z","iopub.status.idle":"2022-08-04T15:24:28.339617Z","shell.execute_reply.started":"2022-08-04T15:24:28.322993Z","shell.execute_reply":"2022-08-04T15:24:28.338750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T15:24:31.024388Z","iopub.execute_input":"2022-08-04T15:24:31.024842Z","iopub.status.idle":"2022-08-04T15:24:31.510047Z","shell.execute_reply.started":"2022-08-04T15:24:31.024806Z","shell.execute_reply":"2022-08-04T15:24:31.508962Z"},"trusted":true},"execution_count":null,"outputs":[]}]}