{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# SALES DATA ANALYSIS AND PROCESSING\n\n## Data source\nThe sales data comes from Kaggle 'Store Sales - Time Series Forecasting' competition, available here: https://www.kaggle.com/competitions/store-sales-time-series-forecasting\n\n## Goal of the competition\nIn this “getting started” competition, you’ll use time-series forecasting to forecast store sales on data from Corporación Favorita, a large Ecuadorian-based grocery retailer.\n\n## This notebook\nIn this notebook, data is imported, pre-processed and analysed. \n\nFollowing methods and plots are employed to analyse the data:\n- Interactive map of the shops, to understand the geographical distribution of the shops\n- Line plots of (a) oil price data, (b) sales vs oil price data, to understand trends in the data\n- Autocorrelation, autocorrelation function and partial autocorrelation plots, to understand trends, seasonality, lags etc.\n- Predictions using autoregression\n\n## Credits\nThis notebook is partially based on this notebook by ALEKSANDR MOROZOV123 user on Kaggle, link here: \n- KAGGLE:https://www.kaggle.com/code/aleksandrmorozov123/time-series-forecasting-with-python\n- GITHUB: https://github.com/AleksandrMorozov123/Competition-on-kaggle-/blob/f82152769d8e1e056d82a0ca89c87d19d7cd5941/time-series-forecasting-with-python.ipynb\n\nOther credits & useful links:\n- Creating interactive notebook: https://vverde.github.io/blob/interactivechoropleth.html\n- Autocorrelation & partial correlation: https://statisticsbyjim.com/time-series/autocorrelation-partial-autocorrelation/\n","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:09:41.445383Z","iopub.execute_input":"2022-07-10T17:09:41.445829Z","iopub.status.idle":"2022-07-10T17:09:41.478984Z","shell.execute_reply.started":"2022-07-10T17:09:41.445741Z","shell.execute_reply":"2022-07-10T17:09:41.477807Z"}}},{"cell_type":"markdown","source":"# IMPORT LIBRARIES","metadata":{}},{"cell_type":"code","source":"import geopandas as gpd\nimport folium \nimport matplotlib.pyplot as plt\nimport pandas as pd\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\nimport seaborn as sns\nsns.set_theme(style=\"darkgrid\")\nimport math\nfrom plotly import __version__\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport cufflinks as cf\ninit_notebook_mode(connected=True)\ncf.go_offline()\n\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:02.166709Z","iopub.execute_input":"2022-07-10T17:10:02.167191Z","iopub.status.idle":"2022-07-10T17:10:04.860436Z","shell.execute_reply.started":"2022-07-10T17:10:02.167153Z","shell.execute_reply":"2022-07-10T17:10:04.859119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:04.862612Z","iopub.execute_input":"2022-07-10T17:10:04.862973Z","iopub.status.idle":"2022-07-10T17:10:05.671118Z","shell.execute_reply.started":"2022-07-10T17:10:04.862939Z","shell.execute_reply":"2022-07-10T17:10:05.669704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATA IMPORTS","metadata":{}},{"cell_type":"code","source":"holidays_data = pd.read_csv('../input/store-sales-time-series-forecasting/holidays_events.csv')\nprint(holidays_data.shape)\nholidays_data['date'] = pd.to_datetime(holidays_data['date'], format = '%Y.%m.%d')\nholidays_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:05.673891Z","iopub.execute_input":"2022-07-10T17:10:05.675139Z","iopub.status.idle":"2022-07-10T17:10:05.722665Z","shell.execute_reply.started":"2022-07-10T17:10:05.675095Z","shell.execute_reply":"2022-07-10T17:10:05.721488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_data = pd.read_csv('../input/store-sales-time-series-forecasting/transactions.csv')\nprint(transactions_data.shape)\ntransactions_data['date'] = pd.to_datetime(transactions_data['date'], format = '%Y.%m.%d')\ntransactions_data.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:05.725298Z","iopub.execute_input":"2022-07-10T17:10:05.725639Z","iopub.status.idle":"2022-07-10T17:10:05.80368Z","shell.execute_reply.started":"2022-07-10T17:10:05.725607Z","shell.execute_reply":"2022-07-10T17:10:05.802596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data = pd.read_csv('../input/store-sales-time-series-forecasting/oil.csv')\noil_data['date'] = pd.to_datetime(oil_data['date'], format = '%Y.%m.%d')\nprint(oil_data.shape)\n#oil_data[oil_data['date']<'2017-09-01']\noil_data.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:05.8048Z","iopub.execute_input":"2022-07-10T17:10:05.805102Z","iopub.status.idle":"2022-07-10T17:10:05.826694Z","shell.execute_reply.started":"2022-07-10T17:10:05.805075Z","shell.execute_reply":"2022-07-10T17:10:05.82555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stores_data = pd.read_csv('../input/store-sales-time-series-forecasting/stores.csv')\nprint(stores_data.shape)\nstores_data.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:05.827994Z","iopub.execute_input":"2022-07-10T17:10:05.828281Z","iopub.status.idle":"2022-07-10T17:10:05.847065Z","shell.execute_reply.started":"2022-07-10T17:10:05.828255Z","shell.execute_reply":"2022-07-10T17:10:05.845921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('../input/store-sales-time-series-forecasting/train.csv')\nprint(train_data.shape)\ntrain_data['date'] = pd.to_datetime(train_data['date'], format = '%Y.%m.%d')\ntrain_data['Year Month'] = train_data['date'].apply(lambda x:x.strftime('%Y %m'))\ntrain_data = pd.merge(left=train_data,right=stores_data, on='store_nbr',how='left')\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:05.848716Z","iopub.execute_input":"2022-07-10T17:10:05.84941Z","iopub.status.idle":"2022-07-10T17:10:29.238877Z","shell.execute_reply.started":"2022-07-10T17:10:05.849362Z","shell.execute_reply":"2022-07-10T17:10:29.237858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check for empty rows and drop missing values","metadata":{}},{"cell_type":"code","source":"holidays_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.240277Z","iopub.execute_input":"2022-07-10T17:10:29.240644Z","iopub.status.idle":"2022-07-10T17:10:29.249913Z","shell.execute_reply.started":"2022-07-10T17:10:29.240612Z","shell.execute_reply":"2022-07-10T17:10:29.248833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.251722Z","iopub.execute_input":"2022-07-10T17:10:29.252296Z","iopub.status.idle":"2022-07-10T17:10:29.266086Z","shell.execute_reply.started":"2022-07-10T17:10:29.252265Z","shell.execute_reply":"2022-07-10T17:10:29.264876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stores_data.isna().sum()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:29.270207Z","iopub.execute_input":"2022-07-10T17:10:29.270578Z","iopub.status.idle":"2022-07-10T17:10:29.279272Z","shell.execute_reply.started":"2022-07-10T17:10:29.270537Z","shell.execute_reply":"2022-07-10T17:10:29.277985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.280665Z","iopub.execute_input":"2022-07-10T17:10:29.281721Z","iopub.status.idle":"2022-07-10T17:10:29.291724Z","shell.execute_reply.started":"2022-07-10T17:10:29.281671Z","shell.execute_reply":"2022-07-10T17:10:29.290717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping missing values\nprint(oil_data.shape)\noil_data = oil_data.dropna()\nprint(oil_data.shape)\noil_data.count()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.293425Z","iopub.execute_input":"2022-07-10T17:10:29.294092Z","iopub.status.idle":"2022-07-10T17:10:29.310253Z","shell.execute_reply.started":"2022-07-10T17:10:29.294051Z","shell.execute_reply":"2022-07-10T17:10:29.309216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EXPLORATORY DATA ANALYSIS\n\n## PLOTTING INTERACTIVE MAP OF THE SHOPS","metadata":{}},{"cell_type":"code","source":"ec_geo = '../input/stores-sales-ecuador/provs_ec.json'\ngeojson = gpd.read_file(ec_geo)\ngeojson.plot(figsize=(6, 6))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.311598Z","iopub.execute_input":"2022-07-10T17:10:29.312399Z","iopub.status.idle":"2022-07-10T17:10:29.722036Z","shell.execute_reply.started":"2022-07-10T17:10:29.312357Z","shell.execute_reply":"2022-07-10T17:10:29.720989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"equador = folium.Map(location=[-1.831239, -78.183403], zoom_start=7, tiles='cartodbpositron')\nequador","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.723418Z","iopub.execute_input":"2022-07-10T17:10:29.723722Z","iopub.status.idle":"2022-07-10T17:10:29.746707Z","shell.execute_reply.started":"2022-07-10T17:10:29.723695Z","shell.execute_reply":"2022-07-10T17:10:29.745652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _, r in geojson.iterrows():\n    sim_geo = gpd.GeoSeries(r['geometry']).simplify(tolerance=0.001)\n    geo_j = sim_geo.to_json()\n    geo_j = folium.GeoJson(data=geo_j,\n                           style_function=lambda x: {'fillColor': 'orange'})\n    folium.Popup(r['dpa_despro']).add_to(geo_j)\n    geo_j.add_to(equador)\nequador","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.7477Z","iopub.execute_input":"2022-07-10T17:10:29.748173Z","iopub.status.idle":"2022-07-10T17:10:29.882428Z","shell.execute_reply.started":"2022-07-10T17:10:29.748144Z","shell.execute_reply":"2022-07-10T17:10:29.881673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stores_data = stores_data.groupby(['state'],as_index=False).count()\nstores_data['state'] = stores_data['state'].str.upper()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.883805Z","iopub.execute_input":"2022-07-10T17:10:29.884567Z","iopub.status.idle":"2022-07-10T17:10:29.893616Z","shell.execute_reply.started":"2022-07-10T17:10:29.884514Z","shell.execute_reply":"2022-07-10T17:10:29.89227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"geojson.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.894895Z","iopub.execute_input":"2022-07-10T17:10:29.895936Z","iopub.status.idle":"2022-07-10T17:10:29.930436Z","shell.execute_reply.started":"2022-07-10T17:10:29.8959Z","shell.execute_reply":"2022-07-10T17:10:29.929683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#GEOJSON PROCESSING \ngeojson = geojson.to_crs(epsg = 2263)\ngeojson['centroid'] = geojson.centroid\ngeojson.drop(['created_at','updated_at'],inplace=True, axis=1)\ngeojson = pd.merge(left=geojson,right=stores_data,left_on='dpa_despro',right_on='state',how='left')\ngeojson['store_nbr'] = geojson['store_nbr'].fillna(0)\ngeojson = geojson.to_crs(epsg = 4326)\ngeojson['centroid'] = geojson['centroid'].to_crs(epsg=4326)\ngeojson.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:29.931789Z","iopub.execute_input":"2022-07-10T17:10:29.932288Z","iopub.status.idle":"2022-07-10T17:10:30.228248Z","shell.execute_reply.started":"2022-07-10T17:10:29.932257Z","shell.execute_reply":"2022-07-10T17:10:30.227072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Adding markers to the map \n# for _, r in geojson.iterrows():\n#     lat = r['centroid'].y\n#     lon = r['centroid'].x\n#     folium.Marker(location=[lat, lon],\n#         popup='length: {}'.format(r['edad_media'])).add_to(equador)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:30.229756Z","iopub.execute_input":"2022-07-10T17:10:30.230107Z","iopub.status.idle":"2022-07-10T17:10:30.234762Z","shell.execute_reply.started":"2022-07-10T17:10:30.230076Z","shell.execute_reply":"2022-07-10T17:10:30.233566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''This code is an adjusted version of the article Interactive choropleth with Python and Folium, link here:\nhttps://vverde.github.io/blob/interactivechoropleth.html'''\n\nimport branca.colormap as cm\nprint(geojson['store_nbr'].unique())\ncolormap = cm.linear.YlGnBu_09.to_step(data=geojson['store_nbr'], method='quant',n=6)\n\nequador = folium.Map(location=[-1.831239, -78.183403], zoom_start=7, tiles='cartodbpositron') \n\nstyle_function = lambda x: {\"weight\":0.5, \n                            'color':'black',\n                            'fillColor':colormap(x['properties']['store_nbr']), 'fillOpacity':0.75}\n                            \nhighlight_function = lambda x: {'fillColor': '#000000', \n                                'color':'#000000', \n                                'fillOpacity': 0.50, \n                                'weight': 0.1}\nNIL=folium.features.GeoJson(\n        geojson.drop('centroid',axis=1),\n        style_function=style_function,\n        control=False,\n        highlight_function=highlight_function,\n        tooltip=folium.features.GeoJsonTooltip(fields=['store_nbr', 'dpa_despro'],\n        aliases=['Number of stores','State'],\n            style=(\"background-color: white; color: #333333; font-family: arial; font-size: 12px; padding: 10px;\"),\n            sticky=True\n        )\n    )\n\ncolormap.add_to(equador)\nequador.add_child(NIL)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:30.236709Z","iopub.execute_input":"2022-07-10T17:10:30.237178Z","iopub.status.idle":"2022-07-10T17:10:30.317215Z","shell.execute_reply.started":"2022-07-10T17:10:30.237132Z","shell.execute_reply":"2022-07-10T17:10:30.316026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Oil price variations\noil_data['date'] = pd.to_datetime(oil_data['date'], format = '%Y.%m.%d')\noil_data['Year Month'] = oil_data['date'].apply(lambda x:x.strftime('%Y[] %m'))\n\nsns.set(rc = {'figure.figsize':(20,9)})\nsns.lineplot(data = oil_data,x='date',y='dcoilwtico').set_title('Oil prices vs time')\nplt.xticks(rotation=90);","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:30.318598Z","iopub.execute_input":"2022-07-10T17:10:30.318933Z","iopub.status.idle":"2022-07-10T17:10:30.69394Z","shell.execute_reply.started":"2022-07-10T17:10:30.3189Z","shell.execute_reply":"2022-07-10T17:10:30.692807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_data.groupby(['date','store_nbr'],as_index=False).agg({'sales':'sum'})\ndf = pd.merge(df,transactions_data,how='left',left_on=['date','store_nbr'],right_on=['date','store_nbr'])\ndf['transactions'] = df['transactions'].fillna(0)\ndf['Year Month'] = df['date'].apply(lambda x:x.strftime('%Y %m'))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:30.695506Z","iopub.execute_input":"2022-07-10T17:10:30.695841Z","iopub.status.idle":"2022-07-10T17:10:31.418319Z","shell.execute_reply.started":"2022-07-10T17:10:30.695812Z","shell.execute_reply":"2022-07-10T17:10:31.417237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = df.groupby(['Year Month','store_nbr'],as_index=False).agg({'sales' : 'sum', 'transactions' : 'sum'})","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:31.419673Z","iopub.execute_input":"2022-07-10T17:10:31.41999Z","iopub.status.idle":"2022-07-10T17:10:31.442868Z","shell.execute_reply.started":"2022-07-10T17:10:31.419961Z","shell.execute_reply":"2022-07-10T17:10:31.441967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The relationship between number of transactions and oil price\nfig, ax = plt.subplots()\nsns.set(rc = {'figure.figsize':(20,9)})\nsns.lineplot(data = df_all,x='Year Month',y='transactions',ax=ax,label='Transactions').set_title('Amount of transactions')\nplt.xticks(rotation = 90);\nax2 = ax.twinx()\nsns.lineplot(data = oil_data,x='Year Month',y='dcoilwtico',ax=ax2,color='red',label='Oil')\nax2.set_ylabel('Oil price ($)')\nax2.legend(loc=0)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:31.445386Z","iopub.execute_input":"2022-07-10T17:10:31.44656Z","iopub.status.idle":"2022-07-10T17:10:35.96593Z","shell.execute_reply.started":"2022-07-10T17:10:31.446478Z","shell.execute_reply":"2022-07-10T17:10:35.965066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FINDINGS:\n- Total amount of sales and transactions increased in years 2013 - 2017. The number of shops increased by 9 between years 2013 - 20\n- Number of sales and transactions is dependant on the oil price, as explained in competition. This is illustrated in the figure above.","metadata":{}},{"cell_type":"code","source":"''' As suggested in this notebook by XODEUM https://www.kaggle.com/code/xodeum/advanced-store-sales-by-time-series'''\n#Products by family\ndf = train_data.groupby('family').agg({'sales':'sum'}).sort_values(by='sales').reset_index()\nplt.barh(df['family'],width=df['sales'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:35.967221Z","iopub.execute_input":"2022-07-10T17:10:35.968094Z","iopub.status.idle":"2022-07-10T17:10:36.57314Z","shell.execute_reply.started":"2022-07-10T17:10:35.968052Z","shell.execute_reply":"2022-07-10T17:10:36.572016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PRE-PROCESSING","metadata":{}},{"cell_type":"code","source":"# Creating lag plot for the datasets\n'''This is employed folowing '''\nfrom pandas.plotting import lag_plot\nlag_plot(holidays_data['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:36.574273Z","iopub.execute_input":"2022-07-10T17:10:36.574616Z","iopub.status.idle":"2022-07-10T17:10:36.956264Z","shell.execute_reply.started":"2022-07-10T17:10:36.574579Z","shell.execute_reply":"2022-07-10T17:10:36.955001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_plot(oil_data['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:36.958272Z","iopub.execute_input":"2022-07-10T17:10:36.958774Z","iopub.status.idle":"2022-07-10T17:10:37.292211Z","shell.execute_reply.started":"2022-07-10T17:10:36.958725Z","shell.execute_reply":"2022-07-10T17:10:37.291042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lag_plot(transactions_data['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:37.29739Z","iopub.execute_input":"2022-07-10T17:10:37.297886Z","iopub.status.idle":"2022-07-10T17:10:37.777798Z","shell.execute_reply.started":"2022-07-10T17:10:37.297851Z","shell.execute_reply":"2022-07-10T17:10:37.77663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hence, data has linear structure.","metadata":{}},{"cell_type":"markdown","source":"## Plotting partial autocorrelation","metadata":{}},{"cell_type":"code","source":"import matplotlib.dates as mpl_dates\nholidays_data['date'] = holidays_data['date'].apply(mpl_dates.date2num)\nholidays_data['date'] = holidays_data['date'].astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:37.779422Z","iopub.execute_input":"2022-07-10T17:10:37.779766Z","iopub.status.idle":"2022-07-10T17:10:37.795647Z","shell.execute_reply.started":"2022-07-10T17:10:37.779733Z","shell.execute_reply":"2022-07-10T17:10:37.79434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.plotting.autocorrelation_plot(holidays_data['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:37.797016Z","iopub.execute_input":"2022-07-10T17:10:37.797429Z","iopub.status.idle":"2022-07-10T17:10:38.062322Z","shell.execute_reply.started":"2022-07-10T17:10:37.79739Z","shell.execute_reply":"2022-07-10T17:10:38.061068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data['date'] = oil_data['date'].apply(mpl_dates.date2num)\noil_data['date'] = oil_data['date'].astype(float)\npd.plotting.autocorrelation_plot(oil_data['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:38.063748Z","iopub.execute_input":"2022-07-10T17:10:38.064078Z","iopub.status.idle":"2022-07-10T17:10:38.350586Z","shell.execute_reply.started":"2022-07-10T17:10:38.064048Z","shell.execute_reply":"2022-07-10T17:10:38.349489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_data['date'] = transactions_data['date'].apply(mpl_dates.date2num)\ntransactions_data['date'] = transactions_data['date'].astype(float)\npd.plotting.autocorrelation_plot(transactions_data['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:38.352106Z","iopub.execute_input":"2022-07-10T17:10:38.352819Z","iopub.status.idle":"2022-07-10T17:10:47.595879Z","shell.execute_reply.started":"2022-07-10T17:10:38.35278Z","shell.execute_reply":"2022-07-10T17:10:47.594606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ACF helps to assess the properties of time series. \n\nFrom the plots above, it can be deducted that:\n- Time series are not stationary, white-noise signal\n- There is no seasonality in the data","metadata":{}},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_acf\nplot_acf(holidays_data['date']);","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:47.597375Z","iopub.execute_input":"2022-07-10T17:10:47.597834Z","iopub.status.idle":"2022-07-10T17:10:47.857653Z","shell.execute_reply.started":"2022-07-10T17:10:47.597792Z","shell.execute_reply":"2022-07-10T17:10:47.856551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_acf(oil_data['date']);","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-10T17:10:47.859179Z","iopub.execute_input":"2022-07-10T17:10:47.859625Z","iopub.status.idle":"2022-07-10T17:10:48.127707Z","shell.execute_reply.started":"2022-07-10T17:10:47.859581Z","shell.execute_reply":"2022-07-10T17:10:48.126331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_acf(transactions_data['date']);","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:48.129482Z","iopub.execute_input":"2022-07-10T17:10:48.130352Z","iopub.status.idle":"2022-07-10T17:10:49.605568Z","shell.execute_reply.started":"2022-07-10T17:10:48.130306Z","shell.execute_reply":"2022-07-10T17:10:49.604801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_pacf","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:49.606503Z","iopub.execute_input":"2022-07-10T17:10:49.607339Z","iopub.status.idle":"2022-07-10T17:10:49.61158Z","shell.execute_reply.started":"2022-07-10T17:10:49.607305Z","shell.execute_reply":"2022-07-10T17:10:49.610889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PACF only displays the correlation between observations that the shorter lags do not explain. Hence, more useful for specifying autoregressive model.\n\nThe PACF suggests fitting a second autoregressive models (partial correlations for lags 1 and 2 statistically significant in all the data)","metadata":{}},{"cell_type":"code","source":"plot_pacf(holidays_data['date']);","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:49.613035Z","iopub.execute_input":"2022-07-10T17:10:49.613874Z","iopub.status.idle":"2022-07-10T17:10:49.87454Z","shell.execute_reply.started":"2022-07-10T17:10:49.613841Z","shell.execute_reply":"2022-07-10T17:10:49.871959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pacf(oil_data['date']);","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:49.876009Z","iopub.execute_input":"2022-07-10T17:10:49.876447Z","iopub.status.idle":"2022-07-10T17:10:50.143289Z","shell.execute_reply.started":"2022-07-10T17:10:49.876401Z","shell.execute_reply":"2022-07-10T17:10:50.142144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pacf(transactions_data['date']);","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.1448Z","iopub.execute_input":"2022-07-10T17:10:50.145218Z","iopub.status.idle":"2022-07-10T17:10:50.601007Z","shell.execute_reply.started":"2022-07-10T17:10:50.145183Z","shell.execute_reply":"2022-07-10T17:10:50.599675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The lags plot suggests that relation between the amount of sales to lags is linear. Partial autocorrelation suggests that the dependence can be captured using lags 1, 2. Those partial correlations are justifiable. Consumption trends month-by-month are likely to be closely related to each other.","metadata":{}},{"cell_type":"markdown","source":"# Autoregression","metadata":{}},{"cell_type":"code","source":"from statsmodels.tsa.ar_model import AutoReg","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.6027Z","iopub.execute_input":"2022-07-10T17:10:50.6042Z","iopub.status.idle":"2022-07-10T17:10:50.632779Z","shell.execute_reply.started":"2022-07-10T17:10:50.604161Z","shell.execute_reply":"2022-07-10T17:10:50.631869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model0 = AutoReg(holidays_data['date'],1)\nmodel_fit0 = model0.fit()\nmodel_fit0.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.633909Z","iopub.execute_input":"2022-07-10T17:10:50.634477Z","iopub.status.idle":"2022-07-10T17:10:50.657987Z","shell.execute_reply.started":"2022-07-10T17:10:50.634445Z","shell.execute_reply":"2022-07-10T17:10:50.657226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = AutoReg(oil_data['date'],1)\nmodel_fit1 = model1.fit()\nmodel_fit1.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.659644Z","iopub.execute_input":"2022-07-10T17:10:50.660332Z","iopub.status.idle":"2022-07-10T17:10:50.680486Z","shell.execute_reply.started":"2022-07-10T17:10:50.660284Z","shell.execute_reply":"2022-07-10T17:10:50.67956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = AutoReg(transactions_data['date'],1)\nmodel_fit2 = model2.fit()\nmodel_fit2.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.681882Z","iopub.execute_input":"2022-07-10T17:10:50.682174Z","iopub.status.idle":"2022-07-10T17:10:50.72394Z","shell.execute_reply.started":"2022-07-10T17:10:50.682147Z","shell.execute_reply":"2022-07-10T17:10:50.722711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure = model_fit0.plot_predict(120, 490,in_sample=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.725689Z","iopub.execute_input":"2022-07-10T17:10:50.726433Z","iopub.status.idle":"2022-07-10T17:10:50.997944Z","shell.execute_reply.started":"2022-07-10T17:10:50.726388Z","shell.execute_reply":"2022-07-10T17:10:50.996797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure = model_fit1.plot_predict(120, 490)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:50.999679Z","iopub.execute_input":"2022-07-10T17:10:51.000373Z","iopub.status.idle":"2022-07-10T17:10:51.210419Z","shell.execute_reply.started":"2022-07-10T17:10:51.000328Z","shell.execute_reply":"2022-07-10T17:10:51.209183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure = model_fit2.plot_predict(120, 490)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:10:51.211954Z","iopub.execute_input":"2022-07-10T17:10:51.212676Z","iopub.status.idle":"2022-07-10T17:10:51.426349Z","shell.execute_reply.started":"2022-07-10T17:10:51.212631Z","shell.execute_reply":"2022-07-10T17:10:51.425455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure (figsize = (18, 10))\nfig = model_fit0.plot_diagnostics(fig = fig, lags = 30)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T17:11:15.665469Z","iopub.execute_input":"2022-07-10T17:11:15.66666Z","iopub.status.idle":"2022-07-10T17:11:16.452905Z","shell.execute_reply.started":"2022-07-10T17:11:15.666597Z","shell.execute_reply":"2022-07-10T17:11:16.450019Z"},"trusted":true},"execution_count":null,"outputs":[]}]}